forked from Plemya-x/ALR
Initial commit
This commit is contained in:
379
internal/dl/dl.go
Normal file
379
internal/dl/dl.go
Normal file
@ -0,0 +1,379 @@
|
||||
/*
|
||||
* LURE - Linux User REpository
|
||||
* Copyright (C) 2023 Elara Musayelyan
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
// Package dl contains abstractions for downloadingfiles and directories
|
||||
// from various sources.
|
||||
package dl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/purell"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"golang.org/x/crypto/blake2b"
|
||||
"golang.org/x/crypto/blake2s"
|
||||
"golang.org/x/exp/slices"
|
||||
"lure.sh/lure/internal/dlcache"
|
||||
"lure.sh/lure/pkg/loggerctx"
|
||||
)
|
||||
|
||||
const manifestFileName = ".lure_cache_manifest"
|
||||
|
||||
// ErrChecksumMismatch occurs when the checksum of a downloaded file
|
||||
// does not match the expected checksum provided in the Options struct.
|
||||
var (
|
||||
ErrChecksumMismatch = errors.New("dl: checksums did not match")
|
||||
ErrNoSuchHashAlgo = errors.New("dl: invalid hashing algorithm")
|
||||
)
|
||||
|
||||
// Downloaders contains all the downloaders in the order in which
|
||||
// they should be checked
|
||||
var Downloaders = []Downloader{
|
||||
GitDownloader{},
|
||||
TorrentDownloader{},
|
||||
FileDownloader{},
|
||||
}
|
||||
|
||||
// Type represents the type of download (file or directory)
|
||||
type Type uint8
|
||||
|
||||
const (
|
||||
TypeFile Type = iota
|
||||
TypeDir
|
||||
)
|
||||
|
||||
func (t Type) String() string {
|
||||
switch t {
|
||||
case TypeFile:
|
||||
return "file"
|
||||
case TypeDir:
|
||||
return "dir"
|
||||
}
|
||||
return "<unknown>"
|
||||
}
|
||||
|
||||
// Options contains the options for downloading
|
||||
// files and directories
|
||||
type Options struct {
|
||||
Hash []byte
|
||||
HashAlgorithm string
|
||||
Name string
|
||||
URL string
|
||||
Destination string
|
||||
CacheDisabled bool
|
||||
PostprocDisabled bool
|
||||
Progress io.Writer
|
||||
LocalDir string
|
||||
}
|
||||
|
||||
func (opts Options) NewHash() (hash.Hash, error) {
|
||||
switch opts.HashAlgorithm {
|
||||
case "", "sha256":
|
||||
return sha256.New(), nil
|
||||
case "sha224":
|
||||
return sha256.New224(), nil
|
||||
case "sha512":
|
||||
return sha512.New(), nil
|
||||
case "sha384":
|
||||
return sha512.New384(), nil
|
||||
case "sha1":
|
||||
return sha1.New(), nil
|
||||
case "md5":
|
||||
return md5.New(), nil
|
||||
case "blake2s-128":
|
||||
return blake2s.New256(nil)
|
||||
case "blake2s-256":
|
||||
return blake2s.New256(nil)
|
||||
case "blake2b-256":
|
||||
return blake2b.New(32, nil)
|
||||
case "blake2b-512":
|
||||
return blake2b.New(64, nil)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrNoSuchHashAlgo, opts.HashAlgorithm)
|
||||
}
|
||||
}
|
||||
|
||||
// Manifest holds information about the type and name
|
||||
// of a downloaded file or directory. It is stored inside
|
||||
// each cache directory for later use.
|
||||
type Manifest struct {
|
||||
Type Type
|
||||
Name string
|
||||
}
|
||||
|
||||
type Downloader interface {
|
||||
// Name returns the name of the downloader
|
||||
Name() string
|
||||
// MatchURL checks if the given URL matches
|
||||
// the downloader.
|
||||
MatchURL(string) bool
|
||||
// Download downloads the object at the URL
|
||||
// provided in the options, to the destination
|
||||
// given in the options. It returns a type,
|
||||
// a name for the downloaded object (this may be empty),
|
||||
// and an error.
|
||||
Download(Options) (Type, string, error)
|
||||
}
|
||||
|
||||
// UpdatingDownloader extends the Downloader interface
|
||||
// with an Update method for protocols such as git, which
|
||||
// allow for incremental updates without changing the URL.
|
||||
type UpdatingDownloader interface {
|
||||
Downloader
|
||||
// Update checks for and performs any
|
||||
// available updates for the object
|
||||
// described in the options. It returns
|
||||
// true if an update was performed, or
|
||||
// false if no update was required.
|
||||
Update(Options) (bool, error)
|
||||
}
|
||||
|
||||
// Download downloads a file or directory using the specified options.
|
||||
// It first gets the appropriate downloader for the URL, then checks
|
||||
// if caching is enabled. If caching is enabled, it attempts to get
|
||||
// the cache directory for the URL and update it if necessary.
|
||||
// If the source is found in the cache, it links it to the destination
|
||||
// using hard links. If the source is not found in the cache,
|
||||
// it downloads the source to a new cache directory and links it
|
||||
// to the destination.
|
||||
func Download(ctx context.Context, opts Options) (err error) {
|
||||
log := loggerctx.From(ctx)
|
||||
normalized, err := normalizeURL(opts.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.URL = normalized
|
||||
|
||||
d := getDownloader(opts.URL)
|
||||
|
||||
if opts.CacheDisabled {
|
||||
_, _, err = d.Download(opts)
|
||||
return err
|
||||
}
|
||||
|
||||
var t Type
|
||||
cacheDir, ok := dlcache.Get(ctx, opts.URL)
|
||||
if ok {
|
||||
var updated bool
|
||||
if d, ok := d.(UpdatingDownloader); ok {
|
||||
log.Info("Source can be updated, updating if required").Str("source", opts.Name).Str("downloader", d.Name()).Send()
|
||||
|
||||
updated, err = d.Update(Options{
|
||||
Hash: opts.Hash,
|
||||
HashAlgorithm: opts.HashAlgorithm,
|
||||
Name: opts.Name,
|
||||
URL: opts.URL,
|
||||
Destination: cacheDir,
|
||||
Progress: opts.Progress,
|
||||
LocalDir: opts.LocalDir,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
m, err := getManifest(cacheDir)
|
||||
if err == nil {
|
||||
t = m.Type
|
||||
|
||||
dest := filepath.Join(opts.Destination, m.Name)
|
||||
ok, err := handleCache(cacheDir, dest, m.Name, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ok && !updated {
|
||||
log.Info("Source found in cache and linked to destination").Str("source", opts.Name).Stringer("type", t).Send()
|
||||
return nil
|
||||
} else if ok {
|
||||
log.Info("Source updated and linked to destination").Str("source", opts.Name).Stringer("type", t).Send()
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
// If we cannot read the manifest,
|
||||
// this cache entry is invalid and
|
||||
// the source must be re-downloaded.
|
||||
err = os.RemoveAll(cacheDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Downloading source").Str("source", opts.Name).Str("downloader", d.Name()).Send()
|
||||
|
||||
cacheDir, err = dlcache.New(ctx, opts.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t, name, err := d.Download(Options{
|
||||
Hash: opts.Hash,
|
||||
HashAlgorithm: opts.HashAlgorithm,
|
||||
Name: opts.Name,
|
||||
URL: opts.URL,
|
||||
Destination: cacheDir,
|
||||
Progress: opts.Progress,
|
||||
LocalDir: opts.LocalDir,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = writeManifest(cacheDir, Manifest{t, name})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dest := filepath.Join(opts.Destination, name)
|
||||
_, err = handleCache(cacheDir, dest, name, t)
|
||||
return err
|
||||
}
|
||||
|
||||
// writeManifest writes the manifest to the specified cache directory.
|
||||
func writeManifest(cacheDir string, m Manifest) error {
|
||||
fl, err := os.Create(filepath.Join(cacheDir, manifestFileName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fl.Close()
|
||||
return msgpack.NewEncoder(fl).Encode(m)
|
||||
}
|
||||
|
||||
// getManifest reads the manifest from the specified cache directory.
|
||||
func getManifest(cacheDir string) (m Manifest, err error) {
|
||||
fl, err := os.Open(filepath.Join(cacheDir, manifestFileName))
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
defer fl.Close()
|
||||
|
||||
err = msgpack.NewDecoder(fl).Decode(&m)
|
||||
return
|
||||
}
|
||||
|
||||
// handleCache links the cache directory or a file within it to the destination
|
||||
func handleCache(cacheDir, dest, name string, t Type) (bool, error) {
|
||||
switch t {
|
||||
case TypeFile:
|
||||
cd, err := os.Open(cacheDir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
names, err := cd.Readdirnames(0)
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
cd.Close()
|
||||
|
||||
if slices.Contains(names, name) {
|
||||
err = os.Link(filepath.Join(cacheDir, name), dest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
case TypeDir:
|
||||
err := linkDir(cacheDir, dest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// linkDir recursively walks through a directory, creating
|
||||
// hard links for each file from the src directory to the
|
||||
// dest directory. If it encounters a directory, it will
|
||||
// create a directory with the same name and permissions
|
||||
// in the dest directory, because hard links cannot be
|
||||
// created for directories.
|
||||
func linkDir(src, dest string) error {
|
||||
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.Name() == manifestFileName {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(src, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newPath := filepath.Join(dest, rel)
|
||||
if info.IsDir() {
|
||||
return os.MkdirAll(newPath, info.Mode())
|
||||
}
|
||||
|
||||
return os.Link(path, newPath)
|
||||
})
|
||||
}
|
||||
|
||||
func getDownloader(u string) Downloader {
|
||||
for _, d := range Downloaders {
|
||||
if d.MatchURL(u) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeURL normalizes a URL string, so that insignificant
|
||||
// differences don't change the hash.
|
||||
func normalizeURL(u string) (string, error) {
|
||||
const normalizationFlags = purell.FlagRemoveTrailingSlash |
|
||||
purell.FlagRemoveDefaultPort |
|
||||
purell.FlagLowercaseHost |
|
||||
purell.FlagLowercaseScheme |
|
||||
purell.FlagRemoveDuplicateSlashes |
|
||||
purell.FlagRemoveFragment |
|
||||
purell.FlagRemoveUnnecessaryHostDots |
|
||||
purell.FlagSortQuery |
|
||||
purell.FlagDecodeHexHost |
|
||||
purell.FlagDecodeOctalHost |
|
||||
purell.FlagDecodeUnnecessaryEscapes |
|
||||
purell.FlagRemoveEmptyPortSeparator
|
||||
|
||||
u, err := purell.NormalizeURLString(u, normalizationFlags)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Fix magnet URLs after normalization
|
||||
u = strings.Replace(u, "magnet://", "magnet:", 1)
|
||||
return u, nil
|
||||
}
|
||||
264
internal/dl/file.go
Normal file
264
internal/dl/file.go
Normal file
@ -0,0 +1,264 @@
|
||||
/*
|
||||
* LURE - Linux User REpository
|
||||
* Copyright (C) 2023 Elara Musayelyan
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package dl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mholt/archiver/v4"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"lure.sh/lure/internal/shutils/handlers"
|
||||
)
|
||||
|
||||
// FileDownloader downloads files using HTTP
|
||||
type FileDownloader struct{}
|
||||
|
||||
// Name always returns "file"
|
||||
func (FileDownloader) Name() string {
|
||||
return "file"
|
||||
}
|
||||
|
||||
// MatchURL always returns true, as FileDownloader
|
||||
// is used as a fallback if nothing else matches
|
||||
func (FileDownloader) MatchURL(string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Download downloads a file using HTTP. If the file is
|
||||
// compressed using a supported format, it will be extracted
|
||||
func (FileDownloader) Download(opts Options) (Type, string, error) {
|
||||
u, err := url.Parse(opts.URL)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
query := u.Query()
|
||||
|
||||
name := query.Get("~name")
|
||||
query.Del("~name")
|
||||
|
||||
archive := query.Get("~archive")
|
||||
query.Del("~archive")
|
||||
|
||||
u.RawQuery = query.Encode()
|
||||
|
||||
var r io.ReadCloser
|
||||
var size int64
|
||||
if u.Scheme == "local" {
|
||||
localFl, err := os.Open(filepath.Join(opts.LocalDir, u.Path))
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
fi, err := localFl.Stat()
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
size = fi.Size()
|
||||
if name == "" {
|
||||
name = fi.Name()
|
||||
}
|
||||
r = localFl
|
||||
} else {
|
||||
res, err := http.Get(u.String())
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
size = res.ContentLength
|
||||
if name == "" {
|
||||
name = getFilename(res)
|
||||
}
|
||||
r = res.Body
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
opts.PostprocDisabled = archive == "false"
|
||||
|
||||
path := filepath.Join(opts.Destination, name)
|
||||
fl, err := os.Create(path)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer fl.Close()
|
||||
|
||||
var bar io.WriteCloser
|
||||
if opts.Progress != nil {
|
||||
bar = progressbar.NewOptions64(
|
||||
size,
|
||||
progressbar.OptionSetDescription(name),
|
||||
progressbar.OptionSetWriter(opts.Progress),
|
||||
progressbar.OptionShowBytes(true),
|
||||
progressbar.OptionSetWidth(10),
|
||||
progressbar.OptionThrottle(65*time.Millisecond),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionOnCompletion(func() {
|
||||
_, _ = io.WriteString(opts.Progress, "\n")
|
||||
}),
|
||||
progressbar.OptionSpinnerType(14),
|
||||
progressbar.OptionFullWidth(),
|
||||
progressbar.OptionSetRenderBlankState(true),
|
||||
)
|
||||
defer bar.Close()
|
||||
} else {
|
||||
bar = handlers.NopRWC{}
|
||||
}
|
||||
|
||||
h, err := opts.NewHash()
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
var w io.Writer
|
||||
if opts.Hash != nil {
|
||||
w = io.MultiWriter(fl, h, bar)
|
||||
} else {
|
||||
w = io.MultiWriter(fl, bar)
|
||||
}
|
||||
|
||||
_, err = io.Copy(w, r)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
r.Close()
|
||||
|
||||
if opts.Hash != nil {
|
||||
sum := h.Sum(nil)
|
||||
if !bytes.Equal(sum, opts.Hash) {
|
||||
return 0, "", ErrChecksumMismatch
|
||||
}
|
||||
}
|
||||
|
||||
if opts.PostprocDisabled {
|
||||
return TypeFile, name, nil
|
||||
}
|
||||
|
||||
_, err = fl.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
format, ar, err := archiver.Identify(name, fl)
|
||||
if err == archiver.ErrNoMatch {
|
||||
return TypeFile, name, nil
|
||||
} else if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
err = extractFile(ar, format, name, opts)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
err = os.Remove(path)
|
||||
return TypeDir, "", err
|
||||
}
|
||||
|
||||
// extractFile extracts an archive or decompresses a file
|
||||
func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) {
|
||||
fname := format.Name()
|
||||
|
||||
switch format := format.(type) {
|
||||
case archiver.Extractor:
|
||||
err = format.Extract(context.Background(), r, nil, func(ctx context.Context, f archiver.File) error {
|
||||
fr, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fr.Close()
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fm := fi.Mode()
|
||||
|
||||
path := filepath.Join(opts.Destination, f.NameInArchive)
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(path), 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if f.IsDir() {
|
||||
err = os.Mkdir(path, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
outFl, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fm.Perm())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer outFl.Close()
|
||||
|
||||
_, err = io.Copy(outFl, fr)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case archiver.Decompressor:
|
||||
rc, err := format.OpenReader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
path := filepath.Join(opts.Destination, name)
|
||||
path = strings.TrimSuffix(path, fname)
|
||||
|
||||
outFl, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(outFl, rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getFilename attempts to parse the Content-Disposition
|
||||
// HTTP response header and extract a filename. If the
|
||||
// header does not exist, it will use the last element
|
||||
// of the path.
|
||||
func getFilename(res *http.Response) (name string) {
|
||||
_, params, err := mime.ParseMediaType(res.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return path.Base(res.Request.URL.Path)
|
||||
}
|
||||
if filename, ok := params["filename"]; ok {
|
||||
return filename
|
||||
} else {
|
||||
return path.Base(res.Request.URL.Path)
|
||||
}
|
||||
}
|
||||
198
internal/dl/git.go
Normal file
198
internal/dl/git.go
Normal file
@ -0,0 +1,198 @@
|
||||
/*
|
||||
* LURE - Linux User REpository
|
||||
* Copyright (C) 2023 Elara Musayelyan
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package dl
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
)
|
||||
|
||||
// GitDownloader downloads Git repositories
|
||||
type GitDownloader struct{}
|
||||
|
||||
// Name always returns "git"
|
||||
func (GitDownloader) Name() string {
|
||||
return "git"
|
||||
}
|
||||
|
||||
// MatchURL matches any URLs that start with "git+"
|
||||
func (GitDownloader) MatchURL(u string) bool {
|
||||
return strings.HasPrefix(u, "git+")
|
||||
}
|
||||
|
||||
// Download uses git to clone the repository from the specified URL.
|
||||
// It allows specifying the revision, depth and recursion options
|
||||
// via query string
|
||||
func (GitDownloader) Download(opts Options) (Type, string, error) {
|
||||
u, err := url.Parse(opts.URL)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
u.Scheme = strings.TrimPrefix(u.Scheme, "git+")
|
||||
|
||||
query := u.Query()
|
||||
|
||||
rev := query.Get("~rev")
|
||||
query.Del("~rev")
|
||||
|
||||
name := query.Get("~name")
|
||||
query.Del("~name")
|
||||
|
||||
depthStr := query.Get("~depth")
|
||||
query.Del("~depth")
|
||||
|
||||
recursive := query.Get("~recursive")
|
||||
query.Del("~recursive")
|
||||
|
||||
u.RawQuery = query.Encode()
|
||||
|
||||
depth := 0
|
||||
if depthStr != "" {
|
||||
depth, err = strconv.Atoi(depthStr)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
}
|
||||
|
||||
co := &git.CloneOptions{
|
||||
URL: u.String(),
|
||||
Depth: depth,
|
||||
Progress: opts.Progress,
|
||||
RecurseSubmodules: git.NoRecurseSubmodules,
|
||||
}
|
||||
|
||||
if recursive == "true" {
|
||||
co.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth
|
||||
}
|
||||
|
||||
r, err := git.PlainClone(opts.Destination, false, co)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
err = r.Fetch(&git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"+refs/*:refs/*"},
|
||||
})
|
||||
if err != git.NoErrAlreadyUpToDate && err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
if rev != "" {
|
||||
h, err := r.ResolveRevision(plumbing.Revision(rev))
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
w, err := r.Worktree()
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
err = w.Checkout(&git.CheckoutOptions{
|
||||
Hash: *h,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
}
|
||||
|
||||
if name == "" {
|
||||
name = strings.TrimSuffix(path.Base(u.Path), ".git")
|
||||
}
|
||||
|
||||
return TypeDir, name, nil
|
||||
}
|
||||
|
||||
// Update uses git to pull the repository and update it
|
||||
// to the latest revision. It allows specifying the depth
|
||||
// and recursion options via query string. It returns
|
||||
// true if update was successful and false if the
|
||||
// repository is already up-to-date
|
||||
func (GitDownloader) Update(opts Options) (bool, error) {
|
||||
u, err := url.Parse(opts.URL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
u.Scheme = strings.TrimPrefix(u.Scheme, "git+")
|
||||
|
||||
query := u.Query()
|
||||
query.Del("~rev")
|
||||
|
||||
depthStr := query.Get("~depth")
|
||||
query.Del("~depth")
|
||||
|
||||
recursive := query.Get("~recursive")
|
||||
query.Del("~recursive")
|
||||
|
||||
u.RawQuery = query.Encode()
|
||||
|
||||
r, err := git.PlainOpen(opts.Destination)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
w, err := r.Worktree()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
depth := 0
|
||||
if depthStr != "" {
|
||||
depth, err = strconv.Atoi(depthStr)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
po := &git.PullOptions{
|
||||
Depth: depth,
|
||||
Progress: opts.Progress,
|
||||
RecurseSubmodules: git.NoRecurseSubmodules,
|
||||
}
|
||||
|
||||
if recursive == "true" {
|
||||
po.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth
|
||||
}
|
||||
|
||||
m, err := getManifest(opts.Destination)
|
||||
manifestOK := err == nil
|
||||
|
||||
err = w.Pull(po)
|
||||
if errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if manifestOK {
|
||||
err = writeManifest(opts.Destination, m)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
90
internal/dl/torrent.go
Normal file
90
internal/dl/torrent.go
Normal file
@ -0,0 +1,90 @@
|
||||
package dl
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
urlMatchRegex = regexp.MustCompile(`(magnet|torrent\+https?):.*`)
|
||||
ErrAria2NotFound = errors.New("aria2 must be installed for torrent functionality")
|
||||
ErrDestinationEmpty = errors.New("the destination directory is empty")
|
||||
)
|
||||
|
||||
type TorrentDownloader struct{}
|
||||
|
||||
// Name always returns "file"
|
||||
func (TorrentDownloader) Name() string {
|
||||
return "torrent"
|
||||
}
|
||||
|
||||
// MatchURL returns true if the URL is a magnet link
|
||||
// or an http(s) link with a "torrent+" prefix
|
||||
func (TorrentDownloader) MatchURL(u string) bool {
|
||||
return urlMatchRegex.MatchString(u)
|
||||
}
|
||||
|
||||
// Download downloads a file over the BitTorrent protocol.
|
||||
func (TorrentDownloader) Download(opts Options) (Type, string, error) {
|
||||
aria2Path, err := exec.LookPath("aria2c")
|
||||
if err != nil {
|
||||
return 0, "", ErrAria2NotFound
|
||||
}
|
||||
|
||||
opts.URL = strings.TrimPrefix(opts.URL, "torrent+")
|
||||
|
||||
cmd := exec.Command(aria2Path, "--summary-interval=0", "--log-level=warn", "--seed-time=0", "--dir="+opts.Destination, opts.URL)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("aria2c returned an error: %w", err)
|
||||
}
|
||||
|
||||
err = removeTorrentFiles(opts.Destination)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
return determineType(opts.Destination)
|
||||
}
|
||||
|
||||
func removeTorrentFiles(path string) error {
|
||||
filePaths, err := filepath.Glob(filepath.Join(path, "*.torrent"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, filePath := range filePaths {
|
||||
err = os.Remove(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func determineType(path string) (Type, string, error) {
|
||||
files, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
|
||||
if len(files) > 1 {
|
||||
return TypeDir, "", nil
|
||||
} else if len(files) == 1 {
|
||||
if files[0].IsDir() {
|
||||
return TypeDir, files[0].Name(), nil
|
||||
} else {
|
||||
return TypeFile, files[0].Name(), nil
|
||||
}
|
||||
}
|
||||
|
||||
return 0, "", ErrDestinationEmpty
|
||||
}
|
||||
Reference in New Issue
Block a user