feat(binana): add tokens database

go/app/util/make-samples.go

@@ -0,0 +1,224 @@
+package util
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/thunderbrewhq/binana/go/app"
+	"github.com/thunderbrewhq/binana/go/app/util/dbutil"
+	"github.com/thunderbrewhq/binana/go/db"
+)
+
+type MakeSampleDatabaseParams struct {
+	// A file name that corresponds to a tree of sample files.
+	// Anything in this tree will be collected into the sample database
+	Source string
+
+	// The name of the file to write the database to
+	Output string
+
+	// Sets the format of the database file
+	Format dbutil.DatabaseFormat
+
+	// URLs that maps to the root of the sample tree hierarchy.
+	// Used to generate a list of mirror URLs for sample binaries
+	DirectMirrors []string
+
+	// List of IPFS Gateway URLs
+	// If not empty, a CID for the sample tree will be created,
+	// Actually uploading anything in the sample tree, however,
+	// is outside the scope of this tool
+	IPFSGateways []string
+}
+
+type sample_database struct {
+	writer        *dbutil.Writer[db.Sample]
+	ipfs_tree_cid string
+	buffer        []db.Sample
+}
+
+func (sample_database *sample_database) add(sample db.Sample) (err error) {
+	sample_database.buffer = append(sample_database.buffer, sample)
+	return
+}
+
+func (sample_database *sample_database) Close() (err error) {
+	if err = sample_database.writer.WriteEntries(sample_database.buffer); err != nil {
+		return
+	}
+	err = sample_database.writer.Close()
+	return
+}
+
+func (sample_database *sample_database) make_sample_file(params *MakeSampleDatabaseParams, name, relative_name string) (err error) {
+	var sample db.Sample
+	// infer mime-type from extension
+	switch filepath.Ext(name) {
+	case ".exe":
+		sample.MimeType = "application/vnd.microsoft.portable-executable"
+	case ".pdb":
+		sample.MimeType = "application/x-ms-pdb"
+		// associate the PDB with its EXE
+		sample_exe_name := strings.TrimSuffix(name, ".pdb") + ".exe"
+		if _, err = os.Stat(sample_exe_name); err == nil {
+			sample.Executable, err = hash_file(sample_exe_name)
+			if err != nil {
+				panic(err)
+			}
+		}
+	case ".macho":
+		sample.MimeType = "application/x-mach-binary"
+	case ".elf":
+		sample.MimeType = "application/x-executable"
+	default:
+		// don't care about this
+		return
+	}
+
+	sample.ID, err = hash_file(name)
+	if err != nil {
+		panic(err)
+	}
+
+	// get the base filename
+	base_name := filepath.Base(name)
+
+	// split the base filename without its extension
+	filename_components := strings.Split(strings.TrimSuffix(base_name, filepath.Ext(base_name)), "-")
+	// now, parse the filename (these must be correctly named!)
+	sample.Program = filename_components[0]
+	sample.Version = filename_components[1]
+	var build uint64
+	build, err = strconv.ParseUint(filename_components[2], 0, 64)
+	if err != nil {
+		panic(err)
+	}
+	sample.Build = uint32(build)
+	sample.OS = filename_components[3]
+	sample.Arch = filename_components[4]
+
+	// now, create various mirrors
+	for _, direct_mirror := range params.DirectMirrors {
+		sample.Mirrors = append(sample.Mirrors, db.SampleMirror{
+			Kind: db.MirrorDirect,
+			URL:  direct_mirror + relative_name,
+		})
+	}
+	for _, ipfs_gateway := range params.IPFSGateways {
+		sample.Mirrors = append(sample.Mirrors, db.SampleMirror{
+			Kind: db.MirrorIPFS,
+			URL:  ipfs_gateway + "/" + sample_database.ipfs_tree_cid + relative_name,
+		})
+	}
+
+	// now write the sample
+
+	if err = sample_database.add(sample); err != nil {
+		return
+	}
+	return
+}
+
+func (sample_database *sample_database) make_tree(params *MakeSampleDatabaseParams, name, relative_name string) (err error) {
+	var (
+		tree_entries []os.DirEntry
+	)
+
+	tree_entries, err = os.ReadDir(name)
+	if err != nil {
+		return
+	}
+
+	for _, tree_entry := range tree_entries {
+		if tree_entry.IsDir() {
+			if err = sample_database.make_tree(params, name+"/"+tree_entry.Name(), relative_name+"/"+tree_entry.Name()); err != nil {
+				return
+			}
+		} else {
+			if err = sample_database.make_sample_file(params, name+"/"+tree_entry.Name(), relative_name+"/"+tree_entry.Name()); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+func ipfs_generate_file_cid(name string) (cid string, err error) {
+
+	// todo
+	// use command:
+	//   ipfs add -qr --only-hash .
+	// inside the root of the sample tree
+	// the last CID is the root of the tree
+
+	var (
+		wd string
+	)
+	wd, err = os.Getwd()
+	if err != nil {
+		return
+	}
+	err = os.Chdir(name)
+	if err != nil {
+		return
+	}
+
+	command := exec.Command("ipfs", "add", "-qr", "--only-hash", ".")
+	var command_output bytes.Buffer
+	command.Stdout = &command_output
+	command.Run()
+	if command.ProcessState.ExitCode() != 0 {
+		os.Chdir(wd)
+		err = fmt.Errorf("util: ipfs tool exited: %d", command.ProcessState.ExitCode())
+		return
+	}
+
+	// Parse command Output
+	command_output_scanner := bufio.NewScanner(&command_output)
+
+	for command_output_scanner.Scan() {
+		cid = command_output_scanner.Text()
+	}
+
+	err = os.Chdir(wd)
+
+	return
+
+}
+
+func MakeSampleDatabase(params *MakeSampleDatabaseParams) {
+	var (
+		err             error
+		sample_database sample_database
+	)
+
+	// if we want to generate IPFS links, start by getting the CID for the sample tree
+	if len(params.IPFSGateways) != 0 {
+		sample_database.ipfs_tree_cid, err = ipfs_generate_file_cid(params.Source)
+		if err != nil {
+			app.Fatal(err)
+			return
+		}
+	}
+
+	sample_database.writer, err = dbutil.Open[db.Sample](params.Output, params.Format)
+	if err != nil {
+		app.Fatal(err)
+	}
+
+	// make the root tree, with our params, the source as the first tree, and "" (root) as the relative path
+	if err = sample_database.make_tree(params, params.Source, ""); err != nil {
+		app.Fatal(err)
+	}
+
+	if err = sample_database.Close(); err != nil {
+		app.Fatal(err)
+	}
+}