feat: support mirrors

This commit is contained in:
2025-06-19 18:56:24 +03:00
parent 7fa7f8ba82
commit 4c1f2ea90f
8 changed files with 750 additions and 191 deletions

View File

@ -31,7 +31,6 @@ import (
"strings"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
gitConfig "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
@ -69,159 +68,212 @@ func (rs *Repos) Pull(ctx context.Context, repos []types.Repo) error {
}
for _, repo := range repos {
repoURL, err := url.Parse(repo.URL)
urls := []string{repo.URL}
urls = append(urls, repo.Mirrors...)
var lastErr error
for i, repoURL := range urls {
if i > 0 {
slog.Info(gotext.Get("Trying mirror"), "repo", repo.Name, "mirror", repoURL)
}
err := rs.pullRepoFromURL(ctx, repoURL, repo)
if err != nil {
lastErr = err
slog.Warn(gotext.Get("Failed to pull from URL"), "repo", repo.Name, "url", repoURL, "error", err)
continue
}
// Success
return nil
}
return fmt.Errorf("failed to pull repository %s from any URL: %w", repo.Name, lastErr)
}
return nil
}
func readGitRepo(repoDir, repoUrl string) (*git.Repository, bool, error) {
gitDir := filepath.Join(repoDir, ".git")
if fi, err := os.Stat(gitDir); err == nil && fi.IsDir() {
r, err := git.PlainOpen(repoDir)
if err == nil {
err = updateRemoteURL(r, repoUrl)
if err == nil {
_, err := r.Head()
if err == nil {
return r, false, nil
}
if errors.Is(err, plumbing.ErrReferenceNotFound) {
return r, true, nil
}
slog.Debug("error getting HEAD, reinitializing...", "err", err)
}
}
slog.Debug("error while reading repo, reinitializing...", "err", err)
}
if err := os.RemoveAll(repoDir); err != nil {
return nil, false, fmt.Errorf("failed to remove repo directory: %w", err)
}
if err := os.MkdirAll(repoDir, 0o755); err != nil {
return nil, false, fmt.Errorf("failed to create repo directory: %w", err)
}
r, err := git.PlainInit(repoDir, false)
if err != nil {
return nil, false, fmt.Errorf("failed to initialize git repo: %w", err)
}
_, err = r.CreateRemote(&gitConfig.RemoteConfig{
Name: git.DefaultRemoteName,
URLs: []string{repoUrl},
})
if err != nil {
return nil, false, err
}
return r, true, nil
}
func (rs *Repos) pullRepoFromURL(ctx context.Context, rawRepoUrl string, repo types.Repo) error {
repoURL, err := url.Parse(rawRepoUrl)
if err != nil {
return fmt.Errorf("invalid URL %s: %w", rawRepoUrl, err)
}
slog.Info(gotext.Get("Pulling repository"), "name", repo.Name)
repoDir := filepath.Join(rs.cfg.GetPaths().RepoDir, repo.Name)
var repoFS billy.Filesystem
r, freshGit, err := readGitRepo(repoDir, repoURL.String())
if err != nil {
return fmt.Errorf("failed to open repo")
}
err = r.FetchContext(ctx, &git.FetchOptions{
Progress: os.Stderr,
Force: true,
})
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}
var old *plumbing.Reference
w, err := r.Worktree()
if err != nil {
return err
}
revHash, err := resolveHash(r, repo.Ref)
if err != nil {
return fmt.Errorf("error resolving hash: %w", err)
}
if !freshGit {
old, err = r.Head()
if err != nil {
return err
}
slog.Info(gotext.Get("Pulling repository"), "name", repo.Name)
repoDir := filepath.Join(rs.cfg.GetPaths().RepoDir, repo.Name)
var repoFS billy.Filesystem
gitDir := filepath.Join(repoDir, ".git")
// Only pull repos that contain valid git repos
if fi, err := os.Stat(gitDir); err == nil && fi.IsDir() {
r, err := git.PlainOpen(repoDir)
if err != nil {
return err
}
err = r.FetchContext(ctx, &git.FetchOptions{
Progress: os.Stderr,
Force: true,
})
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}
w, err := r.Worktree()
if err != nil {
return err
}
old, err := r.Head()
if err != nil {
return err
}
revHash, err := resolveHash(r, repo.Ref)
if err != nil {
return fmt.Errorf("error resolving hash: %w", err)
}
if old.Hash() == *revHash {
slog.Info(gotext.Get("Repository up to date"), "name", repo.Name)
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(revHash.String()),
Force: true,
})
if err != nil {
return err
}
repoFS = w.Filesystem
new, err := r.Head()
if err != nil {
return err
}
// If the DB was not present at startup, that means it's
// empty. In this case, we need to update the DB fully
// rather than just incrementally.
if rs.db.IsEmpty() {
err = rs.processRepoFull(ctx, repo, repoDir)
if err != nil {
return err
}
} else {
err = rs.processRepoChanges(ctx, repo, r, w, old, new)
if err != nil {
return err
}
}
} else {
err = os.RemoveAll(repoDir)
if err != nil {
return err
}
err = os.MkdirAll(repoDir, 0o755)
if err != nil {
return err
}
r, err := git.PlainInit(repoDir, false)
if err != nil {
return err
}
_, err = r.CreateRemote(&gitConfig.RemoteConfig{
Name: git.DefaultRemoteName,
URLs: []string{repoURL.String()},
})
if err != nil {
return err
}
err = r.FetchContext(ctx, &git.FetchOptions{
Progress: os.Stderr,
Force: true,
})
if err != nil {
return err
}
w, err := r.Worktree()
if err != nil {
return err
}
revHash, err := resolveHash(r, repo.Ref)
if err != nil {
return fmt.Errorf("error resolving hash: %w", err)
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(revHash.String()),
Force: true,
})
if err != nil {
return err
}
err = rs.processRepoFull(ctx, repo, repoDir)
if err != nil {
return err
}
repoFS = osfs.New(repoDir)
if old.Hash() == *revHash {
slog.Info(gotext.Get("Repository up to date"), "name", repo.Name)
}
}
fl, err := repoFS.Open("alr-repo.toml")
if err != nil {
slog.Warn(gotext.Get("Git repository does not appear to be a valid ALR repo"), "repo", repo.Name)
continue
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(revHash.String()),
Force: true,
})
if err != nil {
return err
}
repoFS = w.Filesystem
var repoCfg types.RepoConfig
err = toml.NewDecoder(fl).Decode(&repoCfg)
new, err := r.Head()
if err != nil {
return err
}
// If the DB was not present at startup, that means it's
// empty. In this case, we need to update the DB fully
// rather than just incrementally.
if rs.db.IsEmpty() || freshGit {
err = rs.processRepoFull(ctx, repo, repoDir)
if err != nil {
return err
}
fl.Close()
// If the version doesn't have a "v" prefix, it's not a standard version.
// It may be "unknown" or a git version, but either way, there's no way
// to compare it to the repo version, so only compare versions with the "v".
if strings.HasPrefix(config.Version, "v") {
if vercmp.Compare(config.Version, repoCfg.Repo.MinVersion) == -1 {
slog.Warn(gotext.Get("ALR repo's minimum ALR version is greater than the current version. Try updating ALR if something doesn't work."), "repo", repo.Name)
}
} else {
err = rs.processRepoChanges(ctx, repo, r, w, old, new)
if err != nil {
return err
}
}
fl, err := repoFS.Open("alr-repo.toml")
if err != nil {
slog.Warn(gotext.Get("Git repository does not appear to be a valid ALR repo"), "repo", repo.Name)
return nil
}
var repoCfg types.RepoConfig
err = toml.NewDecoder(fl).Decode(&repoCfg)
if err != nil {
return err
}
fl.Close()
// If the version doesn't have a "v" prefix, it's not a standard version.
// It may be "unknown" or a git version, but either way, there's no way
// to compare it to the repo version, so only compare versions with the "v".
if strings.HasPrefix(config.Version, "v") {
if vercmp.Compare(config.Version, repoCfg.Repo.MinVersion) == -1 {
slog.Warn(gotext.Get("ALR repo's minimum ALR version is greater than the current version. Try updating ALR if something doesn't work."), "repo", repo.Name)
}
}
return nil
}
func updateRemoteURL(r *git.Repository, newURL string) error {
cfg, err := r.Config()
if err != nil {
return err
}
remote, ok := cfg.Remotes[git.DefaultRemoteName]
if !ok || len(remote.URLs) == 0 {
return fmt.Errorf("no remote '%s' found", git.DefaultRemoteName)
}
currentURL := remote.URLs[0]
if currentURL == newURL {
return nil
}
slog.Debug("Updating remote URL", "old", currentURL, "new", newURL)
err = r.DeleteRemote(git.DefaultRemoteName)
if err != nil {
return fmt.Errorf("failed to delete old remote: %w", err)
}
_, err = r.CreateRemote(&gitConfig.RemoteConfig{
Name: git.DefaultRemoteName,
URLs: []string{newURL},
})
if err != nil {
return fmt.Errorf("failed to create new remote: %w", err)
}
return nil
}

View File

@ -26,7 +26,6 @@ import (
"testing"
"gitea.plemya-x.ru/Plemya-x/ALR/internal/config"
"gitea.plemya-x.ru/Plemya-x/ALR/internal/db"
database "gitea.plemya-x.ru/Plemya-x/ALR/internal/db"
"gitea.plemya-x.ru/Plemya-x/ALR/internal/repos"
"gitea.plemya-x.ru/Plemya-x/ALR/pkg/types"
@ -35,7 +34,7 @@ import (
type TestEnv struct {
Ctx context.Context
Cfg *TestALRConfig
Db *db.Database
Db *database.Database
}
type TestALRConfig struct {

View File

@ -383,19 +383,27 @@ msgstr ""
msgid "ERROR"
msgstr ""
#: internal/repos/pull.go:77
#: internal/repos/pull.go:78
msgid "Trying mirror"
msgstr ""
#: internal/repos/pull.go:84
msgid "Failed to pull from URL"
msgstr ""
#: internal/repos/pull.go:151
msgid "Pulling repository"
msgstr ""
#: internal/repos/pull.go:113
#: internal/repos/pull.go:188
msgid "Repository up to date"
msgstr ""
#: internal/repos/pull.go:204
#: internal/repos/pull.go:223
msgid "Git repository does not appear to be a valid ALR repo"
msgstr ""
#: internal/repos/pull.go:220
#: internal/repos/pull.go:239
msgid ""
"ALR repo's minimum ALR version is greater than the current version. Try "
"updating ALR if something doesn't work."
@ -461,63 +469,104 @@ msgstr ""
msgid "Pull all repositories that have changed"
msgstr ""
#: repo.go:39
#: repo.go:41
msgid "Manage repos"
msgstr ""
#: repo.go:51 repo.go:269
#: repo.go:55 repo.go:625
msgid "Remove an existing repository"
msgstr ""
#: repo.go:53
#: repo.go:57 repo.go:521
msgid "<name>"
msgstr ""
#: repo.go:83
#: repo.go:102 repo.go:465 repo.go:568
msgid "Repo \"%s\" does not exist"
msgstr ""
#: repo.go:90
#: repo.go:109
msgid "Error removing repo directory"
msgstr ""
#: repo.go:94 repo.go:161 repo.go:219
#: repo.go:113 repo.go:180 repo.go:253 repo.go:316 repo.go:389 repo.go:504
#: repo.go:576
msgid "Error saving config"
msgstr ""
#: repo.go:113
#: repo.go:132
msgid "Error removing packages from database"
msgstr ""
#: repo.go:124 repo.go:239
#: repo.go:143 repo.go:595
msgid "Add a new repository"
msgstr ""
#: repo.go:125
#: repo.go:144 repo.go:270 repo.go:345 repo.go:402
msgid "<name> <url>"
msgstr ""
#: repo.go:150
#: repo.go:169
msgid "Repo \"%s\" already exists"
msgstr ""
#: repo.go:187
#: repo.go:206
msgid "Set the reference of the repository"
msgstr ""
#: repo.go:188
#: repo.go:207
msgid "<name> <ref>"
msgstr ""
#: repo.go:246
#: repo.go:269
msgid "Set the main url of the repository"
msgstr ""
#: repo.go:332
msgid "Manage mirrors of repos"
msgstr ""
#: repo.go:344
msgid "Add a mirror URL to repository"
msgstr ""
#: repo.go:401
msgid "Remove mirror from the repository"
msgstr ""
#: repo.go:420
msgid "Ignore if mirror does not exist"
msgstr ""
#: repo.go:425
msgid "Match partial URL (e.g., github.com instead of full URL)"
msgstr ""
#: repo.go:490
msgid "No mirrors containing \"%s\" found in repo \"%s\""
msgstr ""
#: repo.go:492
msgid "URL \"%s\" does not exist in repo \"%s\""
msgstr ""
#: repo.go:508 repo.go:580
msgid "Removed %d mirrors from repo \"%s\"\n"
msgstr ""
#: repo.go:520
msgid "Remove all mirrors from the repository"
msgstr ""
#: repo.go:602
msgid "Name of the new repo"
msgstr ""
#: repo.go:252
#: repo.go:608
msgid "URL of the new repo"
msgstr ""
#: repo.go:276
#: repo.go:632
msgid "Name of the repo to be deleted"
msgstr ""

View File

@ -5,15 +5,15 @@
msgid ""
msgstr ""
"Project-Id-Version: unnamed project\n"
"PO-Revision-Date: 2025-06-15 16:05+0300\n"
"PO-Revision-Date: 2025-06-19 18:54+0300\n"
"Last-Translator: Maxim Slipenko <maks1ms@alt-gnome.ru>\n"
"Language-Team: Russian\n"
"Language: ru\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n"
"%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n"
"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && "
"n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n"
"X-Generator: Gtranslator 48.0\n"
#: build.go:42
@ -356,8 +356,8 @@ msgid ""
"This command is deprecated and would be removed in the future, use \"%s\" "
"instead!"
msgstr ""
"Эта команда устарела и будет удалена в будущем, используйте вместо нее \"%s"
"\"!"
"Эта команда устарела и будет удалена в будущем, используйте вместо нее "
"\"%s\"!"
#: internal/db/db.go:76
msgid "Database version mismatch; resetting"
@ -397,19 +397,27 @@ msgstr "%s %s загружается — %s/с\n"
msgid "ERROR"
msgstr "ОШИБКА"
#: internal/repos/pull.go:77
#: internal/repos/pull.go:78
msgid "Trying mirror"
msgstr "Пробую зеркало"
#: internal/repos/pull.go:84
msgid "Failed to pull from URL"
msgstr "Не удалось извлечь из URL"
#: internal/repos/pull.go:151
msgid "Pulling repository"
msgstr "Скачивание репозитория"
#: internal/repos/pull.go:113
#: internal/repos/pull.go:188
msgid "Repository up to date"
msgstr "Репозиторий уже обновлён"
#: internal/repos/pull.go:204
#: internal/repos/pull.go:223
msgid "Git repository does not appear to be a valid ALR repo"
msgstr "Репозиторий Git не поддерживается репозиторием ALR"
#: internal/repos/pull.go:220
#: internal/repos/pull.go:239
msgid ""
"ALR repo's minimum ALR version is greater than the current version. Try "
"updating ALR if something doesn't work."
@ -477,63 +485,104 @@ msgstr "Ошибка при запуске приложения"
msgid "Pull all repositories that have changed"
msgstr "Скачать все изменённые репозитории"
#: repo.go:39
#: repo.go:41
msgid "Manage repos"
msgstr "Управление репозиториями"
#: repo.go:51 repo.go:269
#: repo.go:55 repo.go:625
msgid "Remove an existing repository"
msgstr "Удалить существующий репозиторий"
#: repo.go:53
#: repo.go:57 repo.go:521
msgid "<name>"
msgstr "<имя>"
#: repo.go:83
#: repo.go:102 repo.go:465 repo.go:568
msgid "Repo \"%s\" does not exist"
msgstr "Репозитория \"%s\" не существует"
#: repo.go:90
#: repo.go:109
msgid "Error removing repo directory"
msgstr "Ошибка при удалении каталога репозитория"
#: repo.go:94 repo.go:161 repo.go:219
#: repo.go:113 repo.go:180 repo.go:253 repo.go:316 repo.go:389 repo.go:504
#: repo.go:576
msgid "Error saving config"
msgstr "Ошибка при сохранении конфигурации"
#: repo.go:113
#: repo.go:132
msgid "Error removing packages from database"
msgstr "Ошибка при удалении пакетов из базы данных"
#: repo.go:124 repo.go:239
#: repo.go:143 repo.go:595
msgid "Add a new repository"
msgstr "Добавить новый репозиторий"
#: repo.go:125
#: repo.go:144 repo.go:270 repo.go:345 repo.go:402
msgid "<name> <url>"
msgstr "<имя> <url>"
#: repo.go:150
#: repo.go:169
msgid "Repo \"%s\" already exists"
msgstr "Репозиторий \"%s\" уже существует"
#: repo.go:187
#: repo.go:206
msgid "Set the reference of the repository"
msgstr "Установить ссылку на версию репозитория"
#: repo.go:188
#: repo.go:207
msgid "<name> <ref>"
msgstr "<имя> <ссылкааерсию>"
#: repo.go:246
#: repo.go:269
msgid "Set the main url of the repository"
msgstr "Установить главный URL репозитория"
#: repo.go:332
msgid "Manage mirrors of repos"
msgstr "Управление зеркалами репозитория"
#: repo.go:344
msgid "Add a mirror URL to repository"
msgstr "Добавить зеркало репозитория"
#: repo.go:401
msgid "Remove mirror from the repository"
msgstr "Удалить зеркало из репозитория"
#: repo.go:420
msgid "Ignore if mirror does not exist"
msgstr "Игнорировать, если зеркала не существует"
#: repo.go:425
msgid "Match partial URL (e.g., github.com instead of full URL)"
msgstr "Соответствует частичному URL (например, github.com вместо полного URL)"
#: repo.go:490
msgid "No mirrors containing \"%s\" found in repo \"%s\""
msgstr "В репозитории \"%s\" не найдено зеркал, содержащих \"%s\""
#: repo.go:492
msgid "URL \"%s\" does not exist in repo \"%s\""
msgstr "URL \"%s\" не существует в репозитории \"%s\""
#: repo.go:508 repo.go:580
msgid "Removed %d mirrors from repo \"%s\"\n"
msgstr "Удалены зеркала %d из репозитория \"%s\"\n"
#: repo.go:520
msgid "Remove all mirrors from the repository"
msgstr "Удалить все зеркала из репозитория"
#: repo.go:602
msgid "Name of the new repo"
msgstr "Название нового репозитория"
#: repo.go:252
#: repo.go:608
msgid "URL of the new repo"
msgstr "URL-адрес нового репозитория"
#: repo.go:276
#: repo.go:632
msgid "Name of the repo to be deleted"
msgstr "Название репозитория удалён"