Plemya-x/ALR
2
1
Fork 2
Code Issues 8 Pull Requests Actions Packages Projects 1 Releases 17 Wiki Activity

Merge pull request 'refactor(db, config, repos): migrate from functions to struct' (#9) from Maks1mS/ALR:refactor/db into master

Browse Source 
Reviewed-on: #9
This commit is contained in:
Евгений Храмов
2025-01-18 15:27:10 +00:00
parent fb5c875713 e1829c4824
commit 1fb9c6b574
22 changed files with 897 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/repos/find.go
View File

@ -15,7 +15,6 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package repos
import (
@ -24,10 +23,7 @@ import (
"plemya-x.ru/alr/internal/db"
)
// FindPkgs looks for packages matching the inputs inside the database.
// It returns a map that maps the package name input to any packages found for it.
// It also returns a slice that contains the names of all packages that were not found.
func FindPkgs(ctx context.Context, pkgs []string) (map[string][]db.Package, []string, error) {
func (rs *Repos) FindPkgs(ctx context.Context, pkgs []string) (map[string][]db.Package, []string, error) {
found := map[string][]db.Package{}
notFound := []string(nil)
@ -36,7 +32,7 @@ func FindPkgs(ctx context.Context, pkgs []string) (map[string][]db.Package, []st
continue
}
result, err := db.GetPkgs(ctx, "json_array_contains(provides, ?)", pkgName)
result, err := rs.db.GetPkgs(ctx, "json_array_contains(provides, ?)", pkgName)
if err != nil {
return nil, nil, err
}
@ -55,7 +51,7 @@ func FindPkgs(ctx context.Context, pkgs []string) (map[string][]db.Package, []st
result.Close()
if added == 0 {
result, err := db.GetPkgs(ctx, "name LIKE ?", pkgName)
result, err := rs.db.GetPkgs(ctx, "name LIKE ?", pkgName)
if err != nil {
return nil, nil, err
}

56
pkg/repos/find_test.go
View File

@ -19,7 +19,6 @@
package repos_test
import (
"context"
"reflect"
"strings"
"testing"
@ -30,18 +29,15 @@ import (
)
func TestFindPkgs(t *testing.T) {
_, err := db.Open(":memory:")
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
defer db.Close()
e := prepare(t)
defer cleanup(t, e)
setCfgDirs(t)
defer removeCacheDir(t)
rs := repos.New(
e.Cfg,
e.Db,
)
ctx := context.Background()
err = repos.Pull(ctx, []types.Repo{
err := rs.Pull(e.Ctx, []types.Repo{
{
Name: "default",
URL: "https://gitea.plemya-x.ru/xpamych/xpamych-alr-repo.git",
@ -51,7 +47,10 @@ func TestFindPkgs(t *testing.T) {
t.Fatalf("Expected no error, got %s", err)
}
found, notFound, err := repos.FindPkgs([]string{"itd", "nonexistentpackage1", "nonexistentpackage2"})
found, notFound, err := rs.FindPkgs(
e.Ctx,
[]string{"alr", "nonexistentpackage1", "nonexistentpackage2"},
)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
@ -64,33 +63,32 @@ func TestFindPkgs(t *testing.T) {
t.Errorf("Expected 1 package found, got %d", len(found))
}
itdPkgs, ok := found["itd"]
alrPkgs, ok := found["alr"]
if !ok {
t.Fatalf("Expected 'itd' packages to be found")
t.Fatalf("Expected 'alr' packages to be found")
}
if len(itdPkgs) < 2 {
t.Errorf("Expected two 'itd' packages to be found")
if len(alrPkgs) < 2 {
t.Errorf("Expected two 'alr' packages to be found")
}
for i, pkg := range itdPkgs {
if !strings.HasPrefix(pkg.Name, "itd") {
t.Errorf("Expected package name of all found packages to start with 'itd', got %s on element %d", pkg.Name, i)
for i, pkg := range alrPkgs {
if !strings.HasPrefix(pkg.Name, "alr") {
t.Errorf("Expected package name of all found packages to start with 'alr', got %s on element %d", pkg.Name, i)
}
}
}
func TestFindPkgsEmpty(t *testing.T) {
_, err := db.Open(":memory:")
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
defer db.Close()
e := prepare(t)
defer cleanup(t, e)
setCfgDirs(t)
defer removeCacheDir(t)
rs := repos.New(
e.Cfg,
e.Db,
)
err = db.InsertPackage(db.Package{
err := e.Db.InsertPackage(e.Ctx, db.Package{
Name: "test1",
Repository: "default",
Version: "0.0.1",
@ -105,7 +103,7 @@ func TestFindPkgsEmpty(t *testing.T) {
t.Fatalf("Expected no error, got %s", err)
}
err = db.InsertPackage(db.Package{
err = e.Db.InsertPackage(e.Ctx, db.Package{
Name: "test2",
Repository: "default",
Version: "0.0.1",
@ -120,7 +118,7 @@ func TestFindPkgsEmpty(t *testing.T) {
t.Fatalf("Expected no error, got %s", err)
}
found, notFound, err := repos.FindPkgs([]string{"test", ""})
found, notFound, err := rs.FindPkgs(e.Ctx, []string{"test", ""})
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}

130
pkg/repos/pull.go
View File

@ -21,41 +21,48 @@ package repos
import (
"context"
"errors"
"io"
"net/url"
"os"
"path/filepath"
"reflect"
"strings"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/format/diff"
"github.com/pelletier/go-toml/v2"
"go.elara.ws/vercmp"
"plemya-x.ru/alr/internal/config"
"plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/shutils/decoder"
"plemya-x.ru/alr/internal/shutils/handlers"
"plemya-x.ru/alr/internal/types"
"plemya-x.ru/alr/pkg/distro"
"plemya-x.ru/alr/pkg/loggerctx"
"mvdan.cc/sh/v3/expand"
"mvdan.cc/sh/v3/interp"
"mvdan.cc/sh/v3/syntax"
"plemya-x.ru/alr/internal/config"
"plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/shutils/handlers"
"plemya-x.ru/alr/internal/types"
"plemya-x.ru/alr/pkg/loggerctx"
)
type actionType uint8
const (
actionDelete actionType = iota
actionUpdate
)
type action struct {
Type actionType
File string
}
// Pull pulls the provided repositories. If a repo doesn't exist, it will be cloned
// and its packages will be written to the DB. If it does exist, it will be pulled.
// In this case, only changed packages will be processed if possible.
// If repos is set to nil, the repos in the ALR config will be used.
func Pull(ctx context.Context, repos []types.Repo) error {
func (rs *Repos) Pull(ctx context.Context, repos []types.Repo) error {
log := loggerctx.From(ctx)
if repos == nil {
repos = config.Config(ctx).Repos
repos = rs.cfg.Repos(ctx)
}
for _, repo := range repos {
@ -95,7 +102,7 @@ func Pull(ctx context.Context, repos []types.Repo) error {
repoFS = w.Filesystem
// Make sure the DB is created even if the repo is up to date
if !errors.Is(err, git.NoErrAlreadyUpToDate) || db.IsEmpty(ctx) {
if !errors.Is(err, git.NoErrAlreadyUpToDate) || rs.db.IsEmpty(ctx) {
new, err := r.Head()
if err != nil {
return err
@ -104,13 +111,13 @@ func Pull(ctx context.Context, repos []types.Repo) error {
// If the DB was not present at startup, that means it's
// empty. In this case, we need to update the DB fully
// rather than just incrementally.
if db.IsEmpty(ctx) {
err = processRepoFull(ctx, repo, repoDir)
if rs.db.IsEmpty(ctx) {
err = rs.processRepoFull(ctx, repo, repoDir)
if err != nil {
return err
}
} else {
err = processRepoChanges(ctx, repo, r, w, old, new)
err = rs.processRepoChanges(ctx, repo, r, w, old, new)
if err != nil {
return err
}
@ -135,7 +142,7 @@ func Pull(ctx context.Context, repos []types.Repo) error {
return err
}
err = processRepoFull(ctx, repo, repoDir)
err = rs.processRepoFull(ctx, repo, repoDir)
if err != nil {
return err
}
@ -169,19 +176,7 @@ func Pull(ctx context.Context, repos []types.Repo) error {
return nil
}
type actionType uint8
const (
actionDelete actionType = iota
actionUpdate
)
type action struct {
Type actionType
File string
}
func processRepoChanges(ctx context.Context, repo types.Repo, r *git.Repository, w *git.Worktree, old, new *plumbing.Reference) error {
func (rs *Repos) processRepoChanges(ctx context.Context, repo types.Repo, r *git.Repository, w *git.Worktree, old, new *plumbing.Reference) error {
oldCommit, err := r.CommitObject(old.Hash())
if err != nil {
return err
@ -275,7 +270,7 @@ func processRepoChanges(ctx context.Context, repo types.Repo, r *git.Repository,
return err
}
err = db.DeletePkgs(ctx, "name = ? AND repository = ?", pkg.Name, repo.Name)
err = rs.db.DeletePkgs(ctx, "name = ? AND repository = ?", pkg.Name, repo.Name)
if err != nil {
return err
}
@ -310,7 +305,7 @@ func processRepoChanges(ctx context.Context, repo types.Repo, r *git.Repository,
resolveOverrides(runner, &pkg)
err = db.InsertPackage(ctx, pkg)
err = rs.db.InsertPackage(ctx, pkg)
if err != nil {
return err
}
@ -320,23 +315,7 @@ func processRepoChanges(ctx context.Context, repo types.Repo, r *git.Repository,
return nil
}
// isValid makes sure the path of the file being updated is valid.
// It checks to make sure the file is not within a nested directory
// and that it is called alr.sh.
func isValid(from, to diff.File) bool {
var path string
if from != nil {
path = from.Path()
}
if to != nil {
path = to.Path()
}
match, _ := filepath.Match("*/*.sh", path)
return match
}
func processRepoFull(ctx context.Context, repo types.Repo, repoDir string) error {
func (rs *Repos) processRepoFull(ctx context.Context, repo types.Repo, repoDir string) error {
glob := filepath.Join(repoDir, "/*/alr.sh")
matches, err := filepath.Glob(glob)
if err != nil {
@ -380,7 +359,7 @@ func processRepoFull(ctx context.Context, repo types.Repo, repoDir string) error
resolveOverrides(runner, &pkg)
err = db.InsertPackage(ctx, pkg)
err = rs.db.InsertPackage(ctx, pkg)
if err != nil {
return err
}
@ -388,54 +367,3 @@ func processRepoFull(ctx context.Context, repo types.Repo, repoDir string) error
return nil
}
func parseScript(ctx context.Context, parser *syntax.Parser, runner *interp.Runner, r io.ReadCloser, pkg *db.Package) error {
defer r.Close()
fl, err := parser.Parse(r, "alr.sh")
if err != nil {
return err
}
runner.Reset()
err = runner.Run(ctx, fl)
if err != nil {
return err
}
d := decoder.New(&distro.OSRelease{}, runner)
d.Overrides = false
d.LikeDistros = false
return d.DecodeVars(pkg)
}
var overridable = map[string]string{
"deps": "Depends",
"build_deps": "BuildDepends",
"desc": "Description",
"homepage": "Homepage",
"maintainer": "Maintainer",
}
func resolveOverrides(runner *interp.Runner, pkg *db.Package) {
pkgVal := reflect.ValueOf(pkg).Elem()
for name, val := range runner.Vars {
for prefix, field := range overridable {
if strings.HasPrefix(name, prefix) {
override := strings.TrimPrefix(name, prefix)
override = strings.TrimPrefix(override, "_")
field := pkgVal.FieldByName(field)
varVal := field.FieldByName("Val")
varType := varVal.Type()
switch varType.Elem().String() {
case "[]string":
varVal.SetMapIndex(reflect.ValueOf(override), reflect.ValueOf(val.List))
case "string":
varVal.SetMapIndex(reflect.ValueOf(override), reflect.ValueOf(val.Str))
}
break
}
}
}
}

115
pkg/repos/pull_test.go
View File

@ -26,69 +26,104 @@ import (
"plemya-x.ru/alr/internal/config"
"plemya-x.ru/alr/internal/db"
database "plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/types"
"plemya-x.ru/alr/pkg/repos"
)
func setCfgDirs(t *testing.T) {
t.Helper()
paths := config.GetPaths()
var err error
paths.CacheDir, err = os.MkdirTemp("/tmp", "alr-pull-test.*")
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
paths.RepoDir = filepath.Join(paths.CacheDir, "repo")
paths.PkgsDir = filepath.Join(paths.CacheDir, "pkgs")
err = os.MkdirAll(paths.RepoDir, 0o755)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
err = os.MkdirAll(paths.PkgsDir, 0o755)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
paths.DBPath = filepath.Join(paths.CacheDir, "db")
type TestEnv struct {
Ctx context.Context
Cfg *TestALRConfig
Db *db.Database
}
func removeCacheDir(t *testing.T) {
t.Helper()
type TestALRConfig struct {
CacheDir string
RepoDir string
PkgsDir string
}
err := os.RemoveAll(config.GetPaths().CacheDir)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
func (c *TestALRConfig) GetPaths(ctx context.Context) *config.Paths {
return &config.Paths{
DBPath: ":memory:",
CacheDir: c.CacheDir,
RepoDir: c.RepoDir,
PkgsDir: c.PkgsDir,
}
}
func TestPull(t *testing.T) {
_, err := db.Open(":memory:")
func (c *TestALRConfig) Repos(ctx context.Context) []types.Repo {
return []types.Repo{}
}
func prepare(t *testing.T) *TestEnv {
t.Helper()
cacheDir, err := os.MkdirTemp("/tmp", "alr-pull-test.*")
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
defer db.Close()
setCfgDirs(t)
defer removeCacheDir(t)
repoDir := filepath.Join(cacheDir, "repo")
err = os.MkdirAll(repoDir, 0o755)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
pkgsDir := filepath.Join(cacheDir, "pkgs")
err = os.MkdirAll(pkgsDir, 0o755)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
cfg := &TestALRConfig{
CacheDir: cacheDir,
RepoDir: repoDir,
PkgsDir: pkgsDir,
}
ctx := context.Background()
err = repos.Pull(ctx, []types.Repo{
db := database.New(cfg)
db.Init(ctx)
return &TestEnv{
Cfg: cfg,
Db: db,
Ctx: ctx,
}
}
func cleanup(t *testing.T, e *TestEnv) {
t.Helper()
err := os.RemoveAll(e.Cfg.CacheDir)
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
e.Db.Close()
}
func TestPull(t *testing.T) {
e := prepare(t)
defer cleanup(t, e)
rs := repos.New(
e.Cfg,
e.Db,
)
err := rs.Pull(e.Ctx, []types.Repo{
{
Name: "default",
URL: "https://gitea.plemya-x.ru/xpamych/ALR.git",
URL: "https://gitea.plemya-x.ru/Plemya-x/xpamych-alr-repo.git",
},
})
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
result, err := db.GetPkgs("name LIKE 'itd%'")
result, err := e.Db.GetPkgs(e.Ctx, "true")
if err != nil {
t.Fatalf("Expected no error, got %s", err)
}
@ -103,7 +138,7 @@ func TestPull(t *testing.T) {
pkgAmt++
}
if pkgAmt < 2 {
t.Errorf("Expected 2 packages to match, got %d", pkgAmt)
if pkgAmt == 0 {
t.Errorf("Expected at least 1 matching package, but got %d", pkgAmt)
}
}

29
pkg/repos/repos.go Normal file
View File

@ -0,0 +1,29 @@
package repos
import (
"context"
"plemya-x.ru/alr/internal/config"
database "plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/types"
)
type Config interface {
GetPaths(ctx context.Context) *config.Paths
Repos(ctx context.Context) []types.Repo
}
type Repos struct {
cfg Config
db *database.Database
}
func New(
cfg Config,
db *database.Database,
) *Repos {
return &Repos{
cfg,
db,
}
}

54
pkg/repos/repos_legacy.go Normal file
View File

@ -0,0 +1,54 @@
package repos
import (
"context"
"sync"
"plemya-x.ru/alr/internal/config"
"plemya-x.ru/alr/internal/db"
database "plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/types"
)
// Pull pulls the provided repositories. If a repo doesn't exist, it will be cloned
// and its packages will be written to the DB. If it does exist, it will be pulled.
// In this case, only changed packages will be processed if possible.
// If repos is set to nil, the repos in the ALR config will be used.
//
// Deprecated: use struct method
func Pull(ctx context.Context, repos []types.Repo) error {
return GetInstance(ctx).Pull(ctx, repos)
}
// FindPkgs looks for packages matching the inputs inside the database.
// It returns a map that maps the package name input to any packages found for it.
// It also returns a slice that contains the names of all packages that were not found.
//
// Deprecated: use struct method
func FindPkgs(ctx context.Context, pkgs []string) (map[string][]db.Package, []string, error) {
return GetInstance(ctx).FindPkgs(ctx, pkgs)
}
// =======================
// FOR LEGACY ONLY
// =======================
var (
reposInstance *Repos
alrConfigOnce sync.Once
)
// Deprecated: For legacy only
func GetInstance(ctx context.Context) *Repos {
alrConfigOnce.Do(func() {
cfg := config.GetInstance(ctx)
db := database.GetInstance(ctx)
reposInstance = New(
cfg,
db,
)
})
return reposInstance
}

83
pkg/repos/utils.go Normal file
View File

@ -0,0 +1,83 @@
package repos
import (
"context"
"io"
"path/filepath"
"reflect"
"strings"
"github.com/go-git/go-git/v5/plumbing/format/diff"
"mvdan.cc/sh/v3/interp"
"mvdan.cc/sh/v3/syntax"
"plemya-x.ru/alr/internal/db"
"plemya-x.ru/alr/internal/shutils/decoder"
"plemya-x.ru/alr/pkg/distro"
)
// isValid makes sure the path of the file being updated is valid.
// It checks to make sure the file is not within a nested directory
// and that it is called alr.sh.
func isValid(from, to diff.File) bool {
var path string
if from != nil {
path = from.Path()
}
if to != nil {
path = to.Path()
}
match, _ := filepath.Match("*/*.sh", path)
return match
}
func parseScript(ctx context.Context, parser *syntax.Parser, runner *interp.Runner, r io.ReadCloser, pkg *db.Package) error {
defer r.Close()
fl, err := parser.Parse(r, "alr.sh")
if err != nil {
return err
}
runner.Reset()
err = runner.Run(ctx, fl)
if err != nil {
return err
}
d := decoder.New(&distro.OSRelease{}, runner)
d.Overrides = false
d.LikeDistros = false
return d.DecodeVars(pkg)
}
var overridable = map[string]string{
"deps": "Depends",
"build_deps": "BuildDepends",
"desc": "Description",
"homepage": "Homepage",
"maintainer": "Maintainer",
}
func resolveOverrides(runner *interp.Runner, pkg *db.Package) {
pkgVal := reflect.ValueOf(pkg).Elem()
for name, val := range runner.Vars {
for prefix, field := range overridable {
if strings.HasPrefix(name, prefix) {
override := strings.TrimPrefix(name, prefix)
override = strings.TrimPrefix(override, "_")
field := pkgVal.FieldByName(field)
varVal := field.FieldByName("Val")
varType := varVal.Type()
switch varType.Elem().String() {
case "[]string":
varVal.SetMapIndex(reflect.ValueOf(override), reflect.ValueOf(val.List))
case "string":
varVal.SetMapIndex(reflect.ValueOf(override), reflect.ValueOf(val.Str))
}
break
}
}
}
}