forked from Plemya-x/ALR
		
	feat: add checksum for git downloader
This commit is contained in:
		
							
								
								
									
										367
									
								
								pkg/dl/dl.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										367
									
								
								pkg/dl/dl.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,367 @@ | ||||
| // This file was originally part of the project "LURE - Linux User REpository", created by Elara Musayelyan. | ||||
| // It has been modified as part of "ALR - Any Linux Repository" by the ALR Authors. | ||||
| // | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| // Пакет dl содержит абстракции для загрузки файлов и каталогов | ||||
| // из различных источников. | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"context" | ||||
| 	"crypto/md5" | ||||
| 	"crypto/sha1" | ||||
| 	"crypto/sha256" | ||||
| 	"crypto/sha512" | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"hash" | ||||
| 	"io" | ||||
| 	"log/slog" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/PuerkitoBio/purell" | ||||
| 	"github.com/leonelquinteros/gotext" | ||||
| 	"github.com/vmihailenco/msgpack/v5" | ||||
| 	"golang.org/x/crypto/blake2b" | ||||
| 	"golang.org/x/crypto/blake2s" | ||||
| 	"golang.org/x/exp/slices" | ||||
| ) | ||||
|  | ||||
| // Константа для имени файла манифеста кэша | ||||
| const manifestFileName = ".alr_cache_manifest" | ||||
|  | ||||
| // Объявление ошибок для несоответствия контрольной суммы и отсутствия алгоритма хеширования | ||||
| var ( | ||||
| 	ErrChecksumMismatch = errors.New("dl: checksums did not match") | ||||
| 	ErrNoSuchHashAlgo   = errors.New("dl: invalid hashing algorithm") | ||||
| ) | ||||
|  | ||||
| // Массив доступных загрузчиков в порядке их проверки | ||||
| var Downloaders = []Downloader{ | ||||
| 	&GitDownloader{}, | ||||
| 	TorrentDownloader{}, | ||||
| 	FileDownloader{}, | ||||
| } | ||||
|  | ||||
| // Тип данных, представляющий тип загрузки (файл или каталог) | ||||
| type Type uint8 | ||||
|  | ||||
| // Объявление констант для типов загрузки | ||||
| const ( | ||||
| 	TypeFile Type = iota | ||||
| 	TypeDir | ||||
| ) | ||||
|  | ||||
| // Метод для получения строки, представляющей тип загрузки | ||||
| func (t Type) String() string { | ||||
| 	switch t { | ||||
| 	case TypeFile: | ||||
| 		return "file" | ||||
| 	case TypeDir: | ||||
| 		return "dir" | ||||
| 	} | ||||
| 	return "<unknown>" | ||||
| } | ||||
|  | ||||
| type DlCache interface { | ||||
| 	Get(context.Context, string) (string, bool) | ||||
| 	New(context.Context, string) (string, error) | ||||
| } | ||||
|  | ||||
| // Структура Options содержит параметры для загрузки файлов и каталогов | ||||
| type Options struct { | ||||
| 	Hash             []byte | ||||
| 	HashAlgorithm    string | ||||
| 	Name             string | ||||
| 	URL              string | ||||
| 	Destination      string | ||||
| 	CacheDisabled    bool | ||||
| 	PostprocDisabled bool | ||||
| 	Progress         io.Writer | ||||
| 	LocalDir         string | ||||
| 	DlCache          DlCache | ||||
| } | ||||
|  | ||||
| // Метод для создания нового хеша на основе указанного алгоритма хеширования | ||||
| func (opts Options) NewHash() (hash.Hash, error) { | ||||
| 	switch opts.HashAlgorithm { | ||||
| 	case "", "sha256": | ||||
| 		return sha256.New(), nil | ||||
| 	case "sha224": | ||||
| 		return sha256.New224(), nil | ||||
| 	case "sha512": | ||||
| 		return sha512.New(), nil | ||||
| 	case "sha384": | ||||
| 		return sha512.New384(), nil | ||||
| 	case "sha1": | ||||
| 		return sha1.New(), nil | ||||
| 	case "md5": | ||||
| 		return md5.New(), nil | ||||
| 	case "blake2s-128": | ||||
| 		return blake2s.New256(nil) | ||||
| 	case "blake2s-256": | ||||
| 		return blake2s.New256(nil) | ||||
| 	case "blake2b-256": | ||||
| 		return blake2b.New(32, nil) | ||||
| 	case "blake2b-512": | ||||
| 		return blake2b.New(64, nil) | ||||
| 	default: | ||||
| 		return nil, fmt.Errorf("%w: %s", ErrNoSuchHashAlgo, opts.HashAlgorithm) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| // Структура Manifest хранит информацию о типе и имени загруженного файла или каталога | ||||
| type Manifest struct { | ||||
| 	Type Type | ||||
| 	Name string | ||||
| } | ||||
|  | ||||
| // Интерфейс Downloader для реализации различных загрузчиков | ||||
| type Downloader interface { | ||||
| 	Name() string | ||||
| 	MatchURL(string) bool | ||||
| 	Download(context.Context, Options) (Type, string, error) | ||||
| } | ||||
|  | ||||
| // Интерфейс UpdatingDownloader расширяет Downloader методом Update | ||||
| type UpdatingDownloader interface { | ||||
| 	Downloader | ||||
| 	Update(Options) (bool, error) | ||||
| } | ||||
|  | ||||
| // Функция Download загружает файл или каталог с использованием указанных параметров | ||||
| func Download(ctx context.Context, opts Options) (err error) { | ||||
| 	normalized, err := normalizeURL(opts.URL) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	opts.URL = normalized | ||||
|  | ||||
| 	d := getDownloader(opts.URL) | ||||
|  | ||||
| 	if opts.CacheDisabled { | ||||
| 		_, _, err = d.Download(ctx, opts) | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	var t Type | ||||
| 	cacheDir, ok := opts.DlCache.Get(ctx, opts.URL) | ||||
| 	if ok { | ||||
| 		var updated bool | ||||
| 		if d, ok := d.(UpdatingDownloader); ok { | ||||
| 			slog.Info( | ||||
| 				gotext.Get("Source can be updated, updating if required"), | ||||
| 				"source", opts.Name, | ||||
| 				"downloader", d.Name(), | ||||
| 			) | ||||
|  | ||||
| 			updated, err = d.Update(Options{ | ||||
| 				Hash:          opts.Hash, | ||||
| 				HashAlgorithm: opts.HashAlgorithm, | ||||
| 				Name:          opts.Name, | ||||
| 				URL:           opts.URL, | ||||
| 				Destination:   cacheDir, | ||||
| 				Progress:      opts.Progress, | ||||
| 				LocalDir:      opts.LocalDir, | ||||
| 			}) | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
| 		} | ||||
|  | ||||
| 		m, err := getManifest(cacheDir) | ||||
| 		if err == nil { | ||||
| 			t = m.Type | ||||
|  | ||||
| 			dest := filepath.Join(opts.Destination, m.Name) | ||||
| 			ok, err := handleCache(cacheDir, dest, m.Name, t) | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
|  | ||||
| 			if ok && !updated { | ||||
| 				slog.Info( | ||||
| 					gotext.Get("Source found in cache and linked to destination"), | ||||
| 					"source", opts.Name, | ||||
| 					"type", t, | ||||
| 				) | ||||
| 				return nil | ||||
| 			} else if ok { | ||||
| 				slog.Info( | ||||
| 					gotext.Get("Source updated and linked to destination"), | ||||
| 					"source", opts.Name, | ||||
| 					"type", t, | ||||
| 				) | ||||
| 				return nil | ||||
| 			} | ||||
| 		} else { | ||||
| 			err = os.RemoveAll(cacheDir) | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	slog.Info(gotext.Get("Downloading source"), "source", opts.Name, "downloader", d.Name()) | ||||
|  | ||||
| 	cacheDir, err = opts.DlCache.New(ctx, opts.URL) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	t, name, err := d.Download(ctx, Options{ | ||||
| 		Hash:          opts.Hash, | ||||
| 		HashAlgorithm: opts.HashAlgorithm, | ||||
| 		Name:          opts.Name, | ||||
| 		URL:           opts.URL, | ||||
| 		Destination:   cacheDir, | ||||
| 		Progress:      opts.Progress, | ||||
| 		LocalDir:      opts.LocalDir, | ||||
| 	}) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	err = writeManifest(cacheDir, Manifest{t, name}) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	dest := filepath.Join(opts.Destination, name) | ||||
| 	_, err = handleCache(cacheDir, dest, name, t) | ||||
| 	return err | ||||
| } | ||||
|  | ||||
| // Функция writeManifest записывает манифест в указанный каталог кэша | ||||
| func writeManifest(cacheDir string, m Manifest) error { | ||||
| 	fl, err := os.Create(filepath.Join(cacheDir, manifestFileName)) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	defer fl.Close() | ||||
| 	return msgpack.NewEncoder(fl).Encode(m) | ||||
| } | ||||
|  | ||||
| // Функция getManifest считывает манифест из указанного каталога кэша | ||||
| func getManifest(cacheDir string) (m Manifest, err error) { | ||||
| 	fl, err := os.Open(filepath.Join(cacheDir, manifestFileName)) | ||||
| 	if err != nil { | ||||
| 		return Manifest{}, err | ||||
| 	} | ||||
| 	defer fl.Close() | ||||
|  | ||||
| 	err = msgpack.NewDecoder(fl).Decode(&m) | ||||
| 	return | ||||
| } | ||||
|  | ||||
| // Функция handleCache создает жесткие ссылки для файлов из каталога кэша в каталог назначения | ||||
| func handleCache(cacheDir, dest, name string, t Type) (bool, error) { | ||||
| 	switch t { | ||||
| 	case TypeFile: | ||||
| 		cd, err := os.Open(cacheDir) | ||||
| 		if err != nil { | ||||
| 			return false, err | ||||
| 		} | ||||
|  | ||||
| 		names, err := cd.Readdirnames(0) | ||||
| 		if err == io.EOF { | ||||
| 			break | ||||
| 		} else if err != nil { | ||||
| 			return false, err | ||||
| 		} | ||||
|  | ||||
| 		cd.Close() | ||||
|  | ||||
| 		if slices.Contains(names, name) { | ||||
| 			err = os.Link(filepath.Join(cacheDir, name), dest) | ||||
| 			if err != nil { | ||||
| 				return false, err | ||||
| 			} | ||||
| 			return true, nil | ||||
| 		} | ||||
| 	case TypeDir: | ||||
| 		err := linkDir(cacheDir, dest) | ||||
| 		if err != nil { | ||||
| 			return false, err | ||||
| 		} | ||||
| 		return true, nil | ||||
| 	} | ||||
| 	return false, nil | ||||
| } | ||||
|  | ||||
| // Функция linkDir рекурсивно создает жесткие ссылки для файлов из каталога src в каталог dest | ||||
| func linkDir(src, dest string) error { | ||||
| 	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
|  | ||||
| 		if info.Name() == manifestFileName { | ||||
| 			return nil | ||||
| 		} | ||||
|  | ||||
| 		rel, err := filepath.Rel(src, path) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
|  | ||||
| 		newPath := filepath.Join(dest, rel) | ||||
| 		if info.IsDir() { | ||||
| 			return os.MkdirAll(newPath, info.Mode()) | ||||
| 		} | ||||
|  | ||||
| 		return os.Link(path, newPath) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| // Функция getDownloader возвращает загрузчик, соответствующий URL | ||||
| func getDownloader(u string) Downloader { | ||||
| 	for _, d := range Downloaders { | ||||
| 		if d.MatchURL(u) { | ||||
| 			return d | ||||
| 		} | ||||
| 	} | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // Функция normalizeURL нормализует строку URL, чтобы незначительные различия не изменяли хеш | ||||
| func normalizeURL(u string) (string, error) { | ||||
| 	const normalizationFlags = purell.FlagRemoveTrailingSlash | | ||||
| 		purell.FlagRemoveDefaultPort | | ||||
| 		purell.FlagLowercaseHost | | ||||
| 		purell.FlagLowercaseScheme | | ||||
| 		purell.FlagRemoveDuplicateSlashes | | ||||
| 		purell.FlagRemoveFragment | | ||||
| 		purell.FlagRemoveUnnecessaryHostDots | | ||||
| 		purell.FlagSortQuery | | ||||
| 		purell.FlagDecodeHexHost | | ||||
| 		purell.FlagDecodeOctalHost | | ||||
| 		purell.FlagDecodeUnnecessaryEscapes | | ||||
| 		purell.FlagRemoveEmptyPortSeparator | ||||
|  | ||||
| 	u, err := purell.NormalizeURLString(u, normalizationFlags) | ||||
| 	if err != nil { | ||||
| 		return "", err | ||||
| 	} | ||||
|  | ||||
| 	// Исправление URL-адресов magnet после нормализации | ||||
| 	u = strings.Replace(u, "magnet://", "magnet:", 1) | ||||
| 	return u, nil | ||||
| } | ||||
							
								
								
									
										176
									
								
								pkg/dl/dl_test.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										176
									
								
								pkg/dl/dl_test.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,176 @@ | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl_test | ||||
|  | ||||
| import ( | ||||
| 	"context" | ||||
| 	"fmt" | ||||
| 	"log" | ||||
| 	"net/http" | ||||
| 	"net/http/httptest" | ||||
| 	"net/http/httputil" | ||||
| 	"net/url" | ||||
| 	"os" | ||||
| 	"path" | ||||
| 	"strings" | ||||
| 	"testing" | ||||
|  | ||||
| 	"github.com/stretchr/testify/assert" | ||||
|  | ||||
| 	"gitea.plemya-x.ru/Plemya-x/ALR/internal/config" | ||||
| 	"gitea.plemya-x.ru/Plemya-x/ALR/pkg/dl" | ||||
| 	"gitea.plemya-x.ru/Plemya-x/ALR/pkg/dlcache" | ||||
| ) | ||||
|  | ||||
| type TestALRConfig struct{} | ||||
|  | ||||
| func (c *TestALRConfig) GetPaths() *config.Paths { | ||||
| 	return &config.Paths{ | ||||
| 		CacheDir: "/tmp", | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestDownloadWithoutCache(t *testing.T) { | ||||
| 	type testCase struct { | ||||
| 		name     string | ||||
| 		path     string | ||||
| 		expected func(*testing.T, error, string) | ||||
| 	} | ||||
|  | ||||
| 	prepareServer := func() *httptest.Server { | ||||
| 		// URL вашего Git-сервера | ||||
| 		gitServerURL, err := url.Parse("https://gitea.plemya-x.ru") | ||||
| 		if err != nil { | ||||
| 			log.Fatalf("Failed to parse git server URL: %v", err) | ||||
| 		} | ||||
|  | ||||
| 		proxy := httputil.NewSingleHostReverseProxy(gitServerURL) | ||||
|  | ||||
| 		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||||
| 			switch { | ||||
| 			case r.URL.Path == "/file-downloader/file": | ||||
| 				w.WriteHeader(http.StatusOK) | ||||
| 				w.Write([]byte("Hello, World!")) | ||||
| 			case strings.HasPrefix(r.URL.Path, "/git-downloader/git"): | ||||
| 				r.URL.Host = gitServerURL.Host | ||||
| 				r.URL.Scheme = gitServerURL.Scheme | ||||
| 				r.Host = gitServerURL.Host | ||||
| 				r.URL.Path, _ = strings.CutPrefix(r.URL.Path, "/git-downloader/git") | ||||
|  | ||||
| 				proxy.ServeHTTP(w, r) | ||||
| 			default: | ||||
| 				w.WriteHeader(http.StatusNotFound) | ||||
| 			} | ||||
| 		})) | ||||
| 	} | ||||
|  | ||||
| 	for _, tc := range []testCase{ | ||||
| 		{ | ||||
| 			name: "simple file download", | ||||
| 			path: "%s/file-downloader/file", | ||||
| 			expected: func(t *testing.T, err error, tmpdir string) { | ||||
| 				assert.NoError(t, err) | ||||
|  | ||||
| 				_, err = os.Stat(path.Join(tmpdir, "file")) | ||||
| 				assert.NoError(t, err) | ||||
| 			}, | ||||
| 		}, | ||||
| 		{ | ||||
| 			name: "git download", | ||||
| 			path: "git+%s/git-downloader/git/Plemya-x/alr-repo", | ||||
| 			expected: func(t *testing.T, err error, tmpdir string) { | ||||
| 				assert.NoError(t, err) | ||||
|  | ||||
| 				_, err = os.Stat(path.Join(tmpdir, "alr-repo.toml")) | ||||
| 				assert.NoError(t, err) | ||||
| 			}, | ||||
| 		}, | ||||
| 	} { | ||||
| 		t.Run(tc.name, func(t *testing.T) { | ||||
| 			server := prepareServer() | ||||
| 			defer server.Close() | ||||
|  | ||||
| 			tmpdir, err := os.MkdirTemp("", "test-download") | ||||
| 			assert.NoError(t, err) | ||||
| 			defer os.RemoveAll(tmpdir) | ||||
|  | ||||
| 			opts := dl.Options{ | ||||
| 				CacheDisabled: true, | ||||
| 				URL:           fmt.Sprintf(tc.path, server.URL), | ||||
| 				Destination:   tmpdir, | ||||
| 			} | ||||
|  | ||||
| 			err = dl.Download(context.Background(), opts) | ||||
|  | ||||
| 			tc.expected(t, err, tmpdir) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestDownloadFileWithCache(t *testing.T) { | ||||
| 	type testCase struct { | ||||
| 		name string | ||||
| 	} | ||||
|  | ||||
| 	for _, tc := range []testCase{ | ||||
| 		{ | ||||
| 			name: "simple download", | ||||
| 		}, | ||||
| 	} { | ||||
| 		t.Run(tc.name, func(t *testing.T) { | ||||
| 			called := 0 | ||||
| 			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||||
| 				switch { | ||||
| 				case r.URL.Path == "/file": | ||||
| 					called += 1 | ||||
| 					w.WriteHeader(http.StatusOK) | ||||
| 					w.Write([]byte("Hello, World!")) | ||||
| 				default: | ||||
| 					w.WriteHeader(http.StatusNotFound) | ||||
| 				} | ||||
| 			})) | ||||
| 			defer server.Close() | ||||
|  | ||||
| 			tmpdir, err := os.MkdirTemp("", "test-download") | ||||
| 			assert.NoError(t, err) | ||||
| 			defer os.RemoveAll(tmpdir) | ||||
|  | ||||
| 			cfg := &TestALRConfig{} | ||||
|  | ||||
| 			opts := dl.Options{ | ||||
| 				CacheDisabled: false, | ||||
| 				URL:           server.URL + "/file", | ||||
| 				Destination:   tmpdir, | ||||
| 				DlCache:       dlcache.New(cfg), | ||||
| 			} | ||||
|  | ||||
| 			outputFile := path.Join(tmpdir, "file") | ||||
|  | ||||
| 			err = dl.Download(context.Background(), opts) | ||||
| 			assert.NoError(t, err) | ||||
| 			_, err = os.Stat(outputFile) | ||||
| 			assert.NoError(t, err) | ||||
|  | ||||
| 			err = os.Remove(outputFile) | ||||
| 			assert.NoError(t, err) | ||||
|  | ||||
| 			err = dl.Download(context.Background(), opts) | ||||
| 			assert.NoError(t, err) | ||||
| 			assert.Equal(t, 1, called) | ||||
| 		}) | ||||
| 	} | ||||
| } | ||||
							
								
								
									
										270
									
								
								pkg/dl/file.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										270
									
								
								pkg/dl/file.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,270 @@ | ||||
| // This file was originally part of the project "LURE - Linux User REpository", created by Elara Musayelyan. | ||||
| // It has been modified as part of "ALR - Any Linux Repository" by the ALR Authors. | ||||
| // | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"bytes" | ||||
| 	"context" | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"mime" | ||||
| 	"net/http" | ||||
| 	"net/url" | ||||
| 	"os" | ||||
| 	"path" | ||||
| 	"path/filepath" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/mholt/archiver/v4" | ||||
| ) | ||||
|  | ||||
| // FileDownloader загружает файлы с использованием HTTP | ||||
| type FileDownloader struct{} | ||||
|  | ||||
| // Name всегда возвращает "file" | ||||
| func (FileDownloader) Name() string { | ||||
| 	return "file" | ||||
| } | ||||
|  | ||||
| // MatchURL всегда возвращает true, так как FileDownloader | ||||
| // используется как резерв, если ничего другого не соответствует | ||||
| func (FileDownloader) MatchURL(string) bool { | ||||
| 	return true | ||||
| } | ||||
|  | ||||
| // Download загружает файл с использованием HTTP. Если файл | ||||
| // сжат в поддерживаемом формате, он будет распакован | ||||
| func (FileDownloader) Download(ctx context.Context, opts Options) (Type, string, error) { | ||||
| 	// Разбор URL | ||||
| 	u, err := url.Parse(opts.URL) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	// Получение параметров запроса | ||||
| 	query := u.Query() | ||||
|  | ||||
| 	// Получение имени файла из параметров запроса | ||||
| 	name := query.Get("~name") | ||||
| 	query.Del("~name") | ||||
|  | ||||
| 	// Получение параметра архивации | ||||
| 	archive := query.Get("~archive") | ||||
| 	query.Del("~archive") | ||||
|  | ||||
| 	// Кодирование измененных параметров запроса обратно в URL | ||||
| 	u.RawQuery = query.Encode() | ||||
|  | ||||
| 	var r io.ReadCloser | ||||
| 	var size int64 | ||||
|  | ||||
| 	// Проверка схемы URL на "local" | ||||
| 	if u.Scheme == "local" { | ||||
| 		localFl, err := os.Open(filepath.Join(opts.LocalDir, u.Path)) | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
| 		fi, err := localFl.Stat() | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
| 		size = fi.Size() | ||||
| 		if name == "" { | ||||
| 			name = fi.Name() | ||||
| 		} | ||||
| 		r = localFl | ||||
| 	} else { | ||||
| 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) | ||||
| 		if err != nil { | ||||
| 			return 0, "", fmt.Errorf("failed to create request: %w", err) | ||||
| 		} | ||||
| 		// Выполнение HTTP GET запроса | ||||
| 		res, err := http.DefaultClient.Do(req) | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
| 		size = res.ContentLength | ||||
| 		if name == "" { | ||||
| 			name = getFilename(res) | ||||
| 		} | ||||
| 		r = res.Body | ||||
| 	} | ||||
| 	defer r.Close() | ||||
|  | ||||
| 	opts.PostprocDisabled = archive == "false" | ||||
|  | ||||
| 	path := filepath.Join(opts.Destination, name) | ||||
| 	fl, err := os.Create(path) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	var out io.WriteCloser | ||||
| 	// Настройка индикатора прогресса | ||||
| 	if opts.Progress != nil { | ||||
| 		out = NewProgressWriter(fl, size, name, opts.Progress) | ||||
| 	} else { | ||||
| 		out = fl | ||||
| 	} | ||||
| 	defer out.Close() | ||||
|  | ||||
| 	h, err := opts.NewHash() | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	var w io.Writer | ||||
| 	// Настройка MultiWriter для записи в файл, хеш и индикатор прогресса | ||||
| 	if opts.Hash != nil { | ||||
| 		w = io.MultiWriter(h, out) | ||||
| 	} else { | ||||
| 		w = io.MultiWriter(out) | ||||
| 	} | ||||
|  | ||||
| 	// Копирование содержимого из источника в файл назначения | ||||
| 	_, err = io.Copy(w, r) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
| 	r.Close() | ||||
|  | ||||
| 	// Проверка контрольной суммы | ||||
| 	if opts.Hash != nil { | ||||
| 		sum := h.Sum(nil) | ||||
| 		if !bytes.Equal(sum, opts.Hash) { | ||||
| 			return 0, "", ErrChecksumMismatch | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	// Проверка необходимости постобработки | ||||
| 	if opts.PostprocDisabled { | ||||
| 		return TypeFile, name, nil | ||||
| 	} | ||||
|  | ||||
| 	_, err = fl.Seek(0, io.SeekStart) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	// Идентификация формата архива | ||||
| 	format, ar, err := archiver.Identify(name, fl) | ||||
| 	if err == archiver.ErrNoMatch { | ||||
| 		return TypeFile, name, nil | ||||
| 	} else if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	// Распаковка архива | ||||
| 	err = extractFile(ar, format, name, opts) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	// Удаление исходного архива | ||||
| 	err = os.Remove(path) | ||||
| 	return TypeDir, "", err | ||||
| } | ||||
|  | ||||
| // extractFile извлекает архив или распаковывает файл | ||||
| func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) { | ||||
| 	fname := format.Name() | ||||
|  | ||||
| 	// Проверка типа формата архива | ||||
| 	switch format := format.(type) { | ||||
| 	case archiver.Extractor: | ||||
| 		// Извлечение файлов из архива | ||||
| 		err = format.Extract(context.Background(), r, nil, func(ctx context.Context, f archiver.File) error { | ||||
| 			fr, err := f.Open() | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
| 			defer fr.Close() | ||||
| 			fi, err := f.Stat() | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
| 			fm := fi.Mode() | ||||
|  | ||||
| 			path := filepath.Join(opts.Destination, f.NameInArchive) | ||||
|  | ||||
| 			err = os.MkdirAll(filepath.Dir(path), 0o755) | ||||
| 			if err != nil { | ||||
| 				return err | ||||
| 			} | ||||
|  | ||||
| 			if f.IsDir() { | ||||
| 				err = os.MkdirAll(path, 0o755) | ||||
| 				if err != nil { | ||||
| 					return err | ||||
| 				} | ||||
| 			} else { | ||||
| 				outFl, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fm.Perm()) | ||||
| 				if err != nil { | ||||
| 					return err | ||||
| 				} | ||||
| 				defer outFl.Close() | ||||
|  | ||||
| 				_, err = io.Copy(outFl, fr) | ||||
| 				return err | ||||
| 			} | ||||
| 			return nil | ||||
| 		}) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 	case archiver.Decompressor: | ||||
| 		// Распаковка сжатого файла | ||||
| 		rc, err := format.OpenReader(r) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		defer rc.Close() | ||||
|  | ||||
| 		path := filepath.Join(opts.Destination, name) | ||||
| 		path = strings.TrimSuffix(path, fname) | ||||
|  | ||||
| 		outFl, err := os.Create(path) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
|  | ||||
| 		_, err = io.Copy(outFl, rc) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // getFilename пытается разобрать заголовок Content-Disposition | ||||
| // HTTP-ответа и извлечь имя файла. Если заголовок отсутствует, | ||||
| // используется последний элемент пути. | ||||
| func getFilename(res *http.Response) (name string) { | ||||
| 	_, params, err := mime.ParseMediaType(res.Header.Get("Content-Disposition")) | ||||
| 	if err != nil { | ||||
| 		return path.Base(res.Request.URL.Path) | ||||
| 	} | ||||
| 	if filename, ok := params["filename"]; ok { | ||||
| 		return filename | ||||
| 	} else { | ||||
| 		return path.Base(res.Request.URL.Path) | ||||
| 	} | ||||
| } | ||||
							
								
								
									
										238
									
								
								pkg/dl/git.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										238
									
								
								pkg/dl/git.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,238 @@ | ||||
| // This file was originally part of the project "LURE - Linux User REpository", created by Elara Musayelyan. | ||||
| // It has been modified as part of "ALR - Any Linux Repository" by the ALR Authors. | ||||
| // | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"bytes" | ||||
| 	"context" | ||||
| 	"encoding/hex" | ||||
| 	"errors" | ||||
| 	"log/slog" | ||||
| 	"net/url" | ||||
| 	"path" | ||||
| 	"strconv" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/go-git/go-git/v5" | ||||
| 	"github.com/go-git/go-git/v5/config" | ||||
| 	"github.com/go-git/go-git/v5/plumbing" | ||||
| ) | ||||
|  | ||||
| // GitDownloader downloads Git repositories | ||||
| type GitDownloader struct{} | ||||
|  | ||||
| // Name always returns "git" | ||||
| func (GitDownloader) Name() string { | ||||
| 	return "git" | ||||
| } | ||||
|  | ||||
| // MatchURL matches any URLs that start with "git+" | ||||
| func (GitDownloader) MatchURL(u string) bool { | ||||
| 	return strings.HasPrefix(u, "git+") | ||||
| } | ||||
|  | ||||
| // Download uses git to clone the repository from the specified URL. | ||||
| // It allows specifying the revision, depth and recursion options | ||||
| // via query string | ||||
| func (d *GitDownloader) Download(ctx context.Context, opts Options) (Type, string, error) { | ||||
| 	u, err := url.Parse(opts.URL) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
| 	u.Scheme = strings.TrimPrefix(u.Scheme, "git+") | ||||
|  | ||||
| 	query := u.Query() | ||||
|  | ||||
| 	rev := query.Get("~rev") | ||||
| 	query.Del("~rev") | ||||
|  | ||||
| 	// Right now, this only affects the return value of name, | ||||
| 	// which will be used by dl_cache. | ||||
| 	// It seems wrong, but for now it's better to leave it as it is. | ||||
| 	name := query.Get("~name") | ||||
| 	query.Del("~name") | ||||
|  | ||||
| 	depthStr := query.Get("~depth") | ||||
| 	query.Del("~depth") | ||||
|  | ||||
| 	recursive := query.Get("~recursive") | ||||
| 	query.Del("~recursive") | ||||
|  | ||||
| 	u.RawQuery = query.Encode() | ||||
|  | ||||
| 	depth := 0 | ||||
| 	if depthStr != "" { | ||||
| 		depth, err = strconv.Atoi(depthStr) | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	co := &git.CloneOptions{ | ||||
| 		URL:               u.String(), | ||||
| 		Depth:             depth, | ||||
| 		Progress:          opts.Progress, | ||||
| 		RecurseSubmodules: git.NoRecurseSubmodules, | ||||
| 	} | ||||
|  | ||||
| 	if recursive == "true" { | ||||
| 		co.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth | ||||
| 	} | ||||
|  | ||||
| 	r, err := git.PlainCloneContext(ctx, opts.Destination, false, co) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	err = r.Fetch(&git.FetchOptions{ | ||||
| 		RefSpecs: []config.RefSpec{"+refs/*:refs/*"}, | ||||
| 	}) | ||||
| 	if err != git.NoErrAlreadyUpToDate && err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	if rev != "" { | ||||
| 		h, err := r.ResolveRevision(plumbing.Revision(rev)) | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
|  | ||||
| 		w, err := r.Worktree() | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
|  | ||||
| 		err = w.Checkout(&git.CheckoutOptions{ | ||||
| 			Hash: *h, | ||||
| 		}) | ||||
| 		if err != nil { | ||||
| 			return 0, "", err | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	err = d.verifyHash(opts) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	if name == "" { | ||||
| 		name = strings.TrimSuffix(path.Base(u.Path), ".git") | ||||
| 	} | ||||
|  | ||||
| 	return TypeDir, name, nil | ||||
| } | ||||
|  | ||||
| func (GitDownloader) verifyHash(opts Options) error { | ||||
| 	if opts.Hash != nil { | ||||
| 		h, err := opts.NewHash() | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
|  | ||||
| 		err = HashDir(opts.Destination, h) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
|  | ||||
| 		sum := h.Sum(nil) | ||||
|  | ||||
| 		slog.Warn("validate checksum", "real", hex.EncodeToString(sum), "expected", hex.EncodeToString(opts.Hash)) | ||||
|  | ||||
| 		if !bytes.Equal(sum, opts.Hash) { | ||||
| 			return ErrChecksumMismatch | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // Update uses git to pull the repository and update it | ||||
| // to the latest revision. It allows specifying the depth | ||||
| // and recursion options via query string. It returns | ||||
| // true if update was successful and false if the | ||||
| // repository is already up-to-date | ||||
| func (d *GitDownloader) Update(opts Options) (bool, error) { | ||||
| 	u, err := url.Parse(opts.URL) | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
| 	u.Scheme = strings.TrimPrefix(u.Scheme, "git+") | ||||
|  | ||||
| 	query := u.Query() | ||||
| 	query.Del("~rev") | ||||
|  | ||||
| 	depthStr := query.Get("~depth") | ||||
| 	query.Del("~depth") | ||||
|  | ||||
| 	recursive := query.Get("~recursive") | ||||
| 	query.Del("~recursive") | ||||
|  | ||||
| 	u.RawQuery = query.Encode() | ||||
|  | ||||
| 	r, err := git.PlainOpen(opts.Destination) | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	w, err := r.Worktree() | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	depth := 0 | ||||
| 	if depthStr != "" { | ||||
| 		depth, err = strconv.Atoi(depthStr) | ||||
| 		if err != nil { | ||||
| 			return false, err | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	po := &git.PullOptions{ | ||||
| 		Depth:             depth, | ||||
| 		Progress:          opts.Progress, | ||||
| 		RecurseSubmodules: git.NoRecurseSubmodules, | ||||
| 	} | ||||
|  | ||||
| 	if recursive == "true" { | ||||
| 		po.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth | ||||
| 	} | ||||
|  | ||||
| 	m, err := getManifest(opts.Destination) | ||||
| 	manifestOK := err == nil | ||||
|  | ||||
| 	err = w.Pull(po) | ||||
| 	if err != nil { | ||||
| 		if errors.Is(err, git.NoErrAlreadyUpToDate) { | ||||
| 			return false, nil | ||||
| 		} | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	err = d.verifyHash(opts) | ||||
| 	if err != nil { | ||||
| 		return false, err | ||||
| 	} | ||||
|  | ||||
| 	if manifestOK { | ||||
| 		err = writeManifest(opts.Destination, m) | ||||
| 	} | ||||
|  | ||||
| 	return true, err | ||||
| } | ||||
							
								
								
									
										183
									
								
								pkg/dl/git_test.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										183
									
								
								pkg/dl/git_test.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,183 @@ | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl_test | ||||
|  | ||||
| import ( | ||||
| 	"context" | ||||
| 	"encoding/hex" | ||||
| 	"os" | ||||
| 	"os/exec" | ||||
| 	"testing" | ||||
|  | ||||
| 	"github.com/stretchr/testify/assert" | ||||
|  | ||||
| 	"gitea.plemya-x.ru/Plemya-x/ALR/pkg/dl" | ||||
| ) | ||||
|  | ||||
| func TestGitDownloaderMatchUrl(t *testing.T) { | ||||
| 	d := dl.GitDownloader{} | ||||
| 	assert.True(t, d.MatchURL("git+https://example.com/org/project.git")) | ||||
| 	assert.False(t, d.MatchURL("https://example.com/org/project.git")) | ||||
| } | ||||
|  | ||||
| func TestGitDownloaderDownload(t *testing.T) { | ||||
| 	d := dl.GitDownloader{} | ||||
|  | ||||
| 	createTempDir := func(t *testing.T, name string) string { | ||||
| 		t.Helper() | ||||
| 		dir, err := os.MkdirTemp("", "test-"+name) | ||||
| 		assert.NoError(t, err) | ||||
| 		t.Cleanup(func() { | ||||
| 			_ = os.RemoveAll(dir) | ||||
| 		}) | ||||
| 		return dir | ||||
| 	} | ||||
|  | ||||
| 	t.Run("simple", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "simple") | ||||
|  | ||||
| 		dlType, name, err := d.Download(context.Background(), dl.Options{ | ||||
| 			URL:         "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git", | ||||
| 			Destination: dest, | ||||
| 		}) | ||||
|  | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.Equal(t, dl.TypeDir, dlType) | ||||
| 		assert.Equal(t, "repo-for-tests", name) | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("with hash", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "with-hash") | ||||
|  | ||||
| 		hsh, err := hex.DecodeString("33c912b855352663550003ca6b948ae3df1f38e2c036f5a85775df5967e143bf") | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		dlType, name, err := d.Download(context.Background(), dl.Options{ | ||||
| 			URL:           "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git?~rev=init&~name=test", | ||||
| 			Destination:   dest, | ||||
| 			Hash:          hsh, | ||||
| 			HashAlgorithm: "sha256", | ||||
| 		}) | ||||
|  | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.Equal(t, dl.TypeDir, dlType) | ||||
| 		assert.Equal(t, "test", name) | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("with hash (checksum mismatch)", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "with-hash-checksum-mismatch") | ||||
|  | ||||
| 		hsh, err := hex.DecodeString("33c912b855352663550003ca6b948ae3df1f38e2c036f5a85775df5967e143bf") | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		_, _, err = d.Download(context.Background(), dl.Options{ | ||||
| 			URL:           "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git", | ||||
| 			Destination:   dest, | ||||
| 			Hash:          hsh, | ||||
| 			HashAlgorithm: "sha256", | ||||
| 		}) | ||||
|  | ||||
| 		assert.ErrorIs(t, err, dl.ErrChecksumMismatch) | ||||
| 	}) | ||||
| } | ||||
|  | ||||
| func TestGitDownloaderUpdate(t *testing.T) { | ||||
| 	d := dl.GitDownloader{} | ||||
|  | ||||
| 	createTempDir := func(t *testing.T, name string) string { | ||||
| 		t.Helper() | ||||
| 		dir, err := os.MkdirTemp("", "test-"+name) | ||||
|  | ||||
| 		assert.NoError(t, err) | ||||
| 		t.Cleanup(func() { | ||||
| 			_ = os.RemoveAll(dir) | ||||
| 		}) | ||||
| 		return dir | ||||
| 	} | ||||
|  | ||||
| 	setupOldRepo := func(t *testing.T, dest string) { | ||||
| 		t.Helper() | ||||
|  | ||||
| 		cmd := exec.Command("git", "clone", "https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git", dest) | ||||
| 		err := cmd.Run() | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		cmd = exec.Command("git", "-C", dest, "reset", "--hard", "init") | ||||
| 		err = cmd.Run() | ||||
| 		assert.NoError(t, err) | ||||
| 	} | ||||
|  | ||||
| 	t.Run("simple", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "update") | ||||
|  | ||||
| 		setupOldRepo(t, dest) | ||||
|  | ||||
| 		cmd := exec.Command("git", "-C", dest, "rev-parse", "HEAD") | ||||
| 		oldHash, err := cmd.Output() | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		updated, err := d.Update(dl.Options{ | ||||
| 			URL:         "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git", | ||||
| 			Destination: dest, | ||||
| 		}) | ||||
|  | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.True(t, updated) | ||||
|  | ||||
| 		cmd = exec.Command("git", "-C", dest, "rev-parse", "HEAD") | ||||
| 		newHash, err := cmd.Output() | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.NotEqual(t, string(oldHash), string(newHash), "Repository should be updated") | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("with hash", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "update") | ||||
|  | ||||
| 		setupOldRepo(t, dest) | ||||
|  | ||||
| 		hsh, err := hex.DecodeString("0dc4f3c68c435d0cd7a5ee960f965815fa9c4ee0571839cdb8f9de56e06f91eb") | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		updated, err := d.Update(dl.Options{ | ||||
| 			URL:           "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git~rev=test-update-git-downloader", | ||||
| 			Destination:   dest, | ||||
| 			Hash:          hsh, | ||||
| 			HashAlgorithm: "sha256", | ||||
| 		}) | ||||
|  | ||||
| 		assert.NoError(t, err) | ||||
| 		assert.True(t, updated) | ||||
| 	}) | ||||
|  | ||||
| 	t.Run("with hash (checksum mismatch)", func(t *testing.T) { | ||||
| 		dest := createTempDir(t, "update") | ||||
|  | ||||
| 		setupOldRepo(t, dest) | ||||
|  | ||||
| 		hsh, err := hex.DecodeString("33c912b855352663550003ca6b948ae3df1f38e2c036f5a85775df5967e143bf") | ||||
| 		assert.NoError(t, err) | ||||
|  | ||||
| 		_, err = d.Update(dl.Options{ | ||||
| 			URL:           "git+https://gitea.plemya-x.ru/Plemya-x/repo-for-tests.git?~rev=test-update-git-downloader", | ||||
| 			Destination:   dest, | ||||
| 			Hash:          hsh, | ||||
| 			HashAlgorithm: "sha256", | ||||
| 		}) | ||||
|  | ||||
| 		assert.ErrorIs(t, err, dl.ErrChecksumMismatch) | ||||
| 	}) | ||||
| } | ||||
							
								
								
									
										247
									
								
								pkg/dl/progress_tui.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										247
									
								
								pkg/dl/progress_tui.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,247 @@ | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"fmt" | ||||
| 	"io" | ||||
| 	"math" | ||||
| 	"os" | ||||
| 	"time" | ||||
|  | ||||
| 	"github.com/charmbracelet/bubbles/progress" | ||||
| 	"github.com/charmbracelet/bubbles/spinner" | ||||
| 	tea "github.com/charmbracelet/bubbletea" | ||||
| 	"github.com/leonelquinteros/gotext" | ||||
| ) | ||||
|  | ||||
| type model struct { | ||||
| 	progress   progress.Model | ||||
| 	spinner    spinner.Model | ||||
| 	percent    float64 | ||||
| 	speed      float64 | ||||
| 	done       bool | ||||
| 	useSpinner bool | ||||
| 	filename   string | ||||
|  | ||||
| 	total      int64 | ||||
| 	downloaded int64 | ||||
| 	elapsed    time.Duration | ||||
| 	remaining  time.Duration | ||||
|  | ||||
| 	width int | ||||
| } | ||||
|  | ||||
| func (m model) Init() tea.Cmd { | ||||
| 	if m.useSpinner { | ||||
| 		return m.spinner.Tick | ||||
| 	} | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { | ||||
| 	if m.done { | ||||
| 		return m, tea.Quit | ||||
| 	} | ||||
|  | ||||
| 	switch msg := msg.(type) { | ||||
| 	case progressUpdate: | ||||
| 		m.percent = msg.percent | ||||
| 		m.speed = msg.speed | ||||
| 		m.downloaded = msg.downloaded | ||||
| 		m.total = msg.total | ||||
| 		m.elapsed = time.Duration(msg.elapsed) * time.Second | ||||
| 		m.remaining = time.Duration(msg.remaining) * time.Second | ||||
| 		if m.percent >= 1.0 { | ||||
| 			m.done = true | ||||
| 			return m, tea.Quit | ||||
| 		} | ||||
| 		return m, nil | ||||
| 	case tea.WindowSizeMsg: | ||||
| 		m.width = msg.Width | ||||
| 		return m, nil | ||||
| 	case progress.FrameMsg: | ||||
| 		if !m.useSpinner { | ||||
| 			progressModel, cmd := m.progress.Update(msg) | ||||
| 			m.progress = progressModel.(progress.Model) | ||||
| 			return m, cmd | ||||
| 		} | ||||
| 	case spinner.TickMsg: | ||||
| 		if m.useSpinner { | ||||
| 			spinnerModel, cmd := m.spinner.Update(msg) | ||||
| 			m.spinner = spinnerModel | ||||
| 			return m, cmd | ||||
| 		} | ||||
| 	case tea.KeyMsg: | ||||
| 		if msg.String() == "q" { | ||||
| 			return m, tea.Quit | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return m, nil | ||||
| } | ||||
|  | ||||
| func (m model) View() string { | ||||
| 	if m.done { | ||||
| 		return gotext.Get("%s: done!\n", m.filename) | ||||
| 	} | ||||
| 	if m.useSpinner { | ||||
| 		return gotext.Get( | ||||
| 			"%s %s downloading at %s/s\n", | ||||
| 			m.filename, | ||||
| 			m.spinner.View(), | ||||
| 			prettyByteSize(int64(m.speed)), | ||||
| 		) | ||||
| 	} | ||||
|  | ||||
| 	leftPart := m.filename | ||||
|  | ||||
| 	rightPart := fmt.Sprintf("%.2f%% (%s/%s, %s/s) [%v:%v]\n", m.percent*100, | ||||
| 		prettyByteSize(m.downloaded), | ||||
| 		prettyByteSize(m.total), | ||||
| 		prettyByteSize(int64(m.speed)), | ||||
| 		m.elapsed, | ||||
| 		m.remaining, | ||||
| 	) | ||||
|  | ||||
| 	m.progress.Width = m.width - len(leftPart) - len(rightPart) - 6 | ||||
| 	bar := m.progress.ViewAs(m.percent) | ||||
| 	return fmt.Sprintf( | ||||
| 		"%s %s %s", | ||||
| 		leftPart, | ||||
| 		bar, | ||||
| 		rightPart, | ||||
| 	) | ||||
| } | ||||
|  | ||||
| func prettyByteSize(b int64) string { | ||||
| 	bf := float64(b) | ||||
| 	for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} { | ||||
| 		if math.Abs(bf) < 1024.0 { | ||||
| 			return fmt.Sprintf("%3.1f%sB", bf, unit) | ||||
| 		} | ||||
| 		bf /= 1024.0 | ||||
| 	} | ||||
| 	return fmt.Sprintf("%.1fYiB", bf) | ||||
| } | ||||
|  | ||||
| type progressUpdate struct { | ||||
| 	percent float64 | ||||
| 	speed   float64 | ||||
| 	total   int64 | ||||
|  | ||||
| 	downloaded int64 | ||||
| 	elapsed    float64 | ||||
| 	remaining  float64 | ||||
| } | ||||
|  | ||||
| type ProgressWriter struct { | ||||
| 	baseWriter   io.WriteCloser | ||||
| 	total        int64 | ||||
| 	downloaded   int64 | ||||
| 	startTime    time.Time | ||||
| 	onProgress   func(progressUpdate) | ||||
| 	lastReported time.Time | ||||
| 	doneChan     chan struct{} | ||||
| } | ||||
|  | ||||
| func (pw *ProgressWriter) Write(p []byte) (int, error) { | ||||
| 	n, err := pw.baseWriter.Write(p) | ||||
| 	if err != nil { | ||||
| 		return n, err | ||||
| 	} | ||||
|  | ||||
| 	pw.downloaded += int64(n) | ||||
| 	now := time.Now() | ||||
| 	elapsed := now.Sub(pw.startTime).Seconds() | ||||
| 	speed := float64(pw.downloaded) / elapsed | ||||
| 	var remaining, percent float64 | ||||
| 	if pw.total > 0 { | ||||
| 		remaining = (float64(pw.total) - float64(pw.downloaded)) / speed | ||||
| 		percent = float64(pw.downloaded) / float64(pw.total) | ||||
| 	} | ||||
|  | ||||
| 	if now.Sub(pw.lastReported) > 100*time.Millisecond { | ||||
| 		pw.onProgress(progressUpdate{ | ||||
| 			percent:    percent, | ||||
| 			speed:      speed, | ||||
| 			total:      pw.total, | ||||
| 			downloaded: pw.downloaded, | ||||
| 			elapsed:    elapsed, | ||||
| 			remaining:  remaining, | ||||
| 		}) | ||||
| 		pw.lastReported = now | ||||
| 	} | ||||
|  | ||||
| 	return n, nil | ||||
| } | ||||
|  | ||||
| func (pw *ProgressWriter) Close() error { | ||||
| 	pw.onProgress(progressUpdate{ | ||||
| 		percent:    1, | ||||
| 		speed:      0, | ||||
| 		downloaded: pw.downloaded, | ||||
| 	}) | ||||
| 	<-pw.doneChan | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func NewProgressWriter(base io.WriteCloser, max int64, filename string, out io.Writer) *ProgressWriter { | ||||
| 	var m *model | ||||
| 	if max == -1 { | ||||
| 		m = &model{ | ||||
| 			spinner:    spinner.New(), | ||||
| 			useSpinner: true, | ||||
| 			filename:   filename, | ||||
| 		} | ||||
| 		m.spinner.Spinner = spinner.Dot | ||||
| 	} else { | ||||
| 		m = &model{ | ||||
| 			progress: progress.New( | ||||
| 				progress.WithDefaultGradient(), | ||||
| 				progress.WithoutPercentage(), | ||||
| 			), | ||||
| 			useSpinner: false, | ||||
| 			filename:   filename, | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	p := tea.NewProgram(m, | ||||
| 		tea.WithInput(nil), | ||||
| 		tea.WithOutput(out), | ||||
| 	) | ||||
|  | ||||
| 	pw := &ProgressWriter{ | ||||
| 		baseWriter: base, | ||||
| 		total:      max, | ||||
| 		startTime:  time.Now(), | ||||
| 		doneChan:   make(chan struct{}), | ||||
| 		onProgress: func(update progressUpdate) { | ||||
| 			p.Send(update) | ||||
| 		}, | ||||
| 	} | ||||
|  | ||||
| 	go func() { | ||||
| 		defer close(pw.doneChan) | ||||
| 		if _, err := p.Run(); err != nil { | ||||
| 			fmt.Fprintf(os.Stderr, "Error running progress writer: %v\n", err) | ||||
| 			os.Exit(1) | ||||
| 		} | ||||
| 	}() | ||||
|  | ||||
| 	return pw | ||||
| } | ||||
							
								
								
									
										110
									
								
								pkg/dl/torrent.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										110
									
								
								pkg/dl/torrent.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,110 @@ | ||||
| // This file was originally part of the project "LURE - Linux User REpository", created by Elara Musayelyan. | ||||
| // It has been modified as part of "ALR - Any Linux Repository" by the ALR Authors. | ||||
| // | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"context" | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"os" | ||||
| 	"os/exec" | ||||
| 	"path/filepath" | ||||
| 	"regexp" | ||||
| 	"strings" | ||||
| ) | ||||
|  | ||||
| var ( | ||||
| 	urlMatchRegex       = regexp.MustCompile(`(magnet|torrent\+https?):.*`) | ||||
| 	ErrAria2NotFound    = errors.New("aria2 must be installed for torrent functionality") | ||||
| 	ErrDestinationEmpty = errors.New("the destination directory is empty") | ||||
| ) | ||||
|  | ||||
| type TorrentDownloader struct{} | ||||
|  | ||||
| // Name always returns "file" | ||||
| func (TorrentDownloader) Name() string { | ||||
| 	return "torrent" | ||||
| } | ||||
|  | ||||
| // MatchURL returns true if the URL is a magnet link | ||||
| // or an http(s) link with a "torrent+" prefix | ||||
| func (TorrentDownloader) MatchURL(u string) bool { | ||||
| 	return urlMatchRegex.MatchString(u) | ||||
| } | ||||
|  | ||||
| // Download downloads a file over the BitTorrent protocol. | ||||
| func (TorrentDownloader) Download(ctx context.Context, opts Options) (Type, string, error) { | ||||
| 	aria2Path, err := exec.LookPath("aria2c") | ||||
| 	if err != nil { | ||||
| 		return 0, "", ErrAria2NotFound | ||||
| 	} | ||||
|  | ||||
| 	opts.URL = strings.TrimPrefix(opts.URL, "torrent+") | ||||
|  | ||||
| 	cmd := exec.CommandContext(ctx, aria2Path, "--summary-interval=0", "--log-level=warn", "--seed-time=0", "--dir="+opts.Destination, opts.URL) | ||||
| 	cmd.Stdout = os.Stdout | ||||
| 	cmd.Stderr = os.Stderr | ||||
| 	err = cmd.Run() | ||||
| 	if err != nil { | ||||
| 		return 0, "", fmt.Errorf("aria2c returned an error: %w", err) | ||||
| 	} | ||||
|  | ||||
| 	err = removeTorrentFiles(opts.Destination) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	return determineType(opts.Destination) | ||||
| } | ||||
|  | ||||
| func removeTorrentFiles(path string) error { | ||||
| 	filePaths, err := filepath.Glob(filepath.Join(path, "*.torrent")) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	for _, filePath := range filePaths { | ||||
| 		err = os.Remove(filePath) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| func determineType(path string) (Type, string, error) { | ||||
| 	files, err := os.ReadDir(path) | ||||
| 	if err != nil { | ||||
| 		return 0, "", err | ||||
| 	} | ||||
|  | ||||
| 	if len(files) > 1 { | ||||
| 		return TypeDir, "", nil | ||||
| 	} else if len(files) == 1 { | ||||
| 		if files[0].IsDir() { | ||||
| 			return TypeDir, files[0].Name(), nil | ||||
| 		} else { | ||||
| 			return TypeFile, files[0].Name(), nil | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return 0, "", ErrDestinationEmpty | ||||
| } | ||||
							
								
								
									
										55
									
								
								pkg/dl/utils.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										55
									
								
								pkg/dl/utils.go
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,55 @@ | ||||
| // ALR - Any Linux Repository | ||||
| // Copyright (C) 2025 The ALR Authors | ||||
| // | ||||
| // This program is free software: you can redistribute it and/or modify | ||||
| // it under the terms of the GNU General Public License as published by | ||||
| // the Free Software Foundation, either version 3 of the License, or | ||||
| // (at your option) any later version. | ||||
| // | ||||
| // This program is distributed in the hope that it will be useful, | ||||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | ||||
| // GNU General Public License for more details. | ||||
| // | ||||
| // You should have received a copy of the GNU General Public License | ||||
| // along with this program.  If not, see <http://www.gnu.org/licenses/>. | ||||
|  | ||||
| package dl | ||||
|  | ||||
| import ( | ||||
| 	"hash" | ||||
| 	"io" | ||||
| 	"os" | ||||
| 	"path/filepath" | ||||
| ) | ||||
|  | ||||
| func HashDir(dirPath string, h hash.Hash) error { | ||||
| 	err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		// Skip .git directory | ||||
| 		if info.IsDir() && info.Name() == ".git" { | ||||
| 			return filepath.SkipDir | ||||
| 		} | ||||
| 		// Skip directories (only process files) | ||||
| 		if !info.Mode().IsRegular() { | ||||
| 			return nil | ||||
| 		} | ||||
| 		// Open file | ||||
| 		f, err := os.Open(path) | ||||
| 		if err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		defer f.Close() | ||||
| 		// Write file content to hasher | ||||
| 		if _, err := io.Copy(h, f); err != nil { | ||||
| 			return err | ||||
| 		} | ||||
| 		return nil | ||||
| 	}) | ||||
| 	if err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return nil | ||||
| } | ||||
		Reference in New Issue
	
	Block a user