Initial compiler code

.gitlab-ci.yml

@@ -0,0 +1,60 @@
+image: golang:1.14.2
+
+variables:
+  REPO_NAME: ci.itzana.me/qtdb/rbxcompile
+
+before_script:
+  - mkdir -p $GOPATH/src/$(dirname $REPO_NAME)
+  - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME
+  - cd $GOPATH/src/$REPO_NAME
+  - echo -e "machine ci.itzana.me login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc
+
+stages:
+  - test
+  - build
+  - deploy
+
+format:
+  stage: test
+  script:
+    - go fmt $(go list ./... | grep -v /vendor/)
+    - go vet $(go list ./... | grep -v /vendor/)
+    - go test -race $(go list ./... | grep -v /vendor/)
+
+compile:
+  stage: build
+  script:
+    - CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o $CI_PROJECT_DIR/rbxcompiler ./cmd/rbxcompiler
+  artifacts:
+    paths:
+      - rbxcompiler
+
+docker-build-release:
+  image: docker:latest
+  stage: deploy
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE/release" .
+    - docker build --pull -t "$CI_REGISTRY_IMAGE/release:$CI_COMMIT_SHA" .
+    - docker push "$CI_REGISTRY_IMAGE/release"
+    - docker push "$CI_REGISTRY_IMAGE/release:$CI_COMMIT_SHA"
+  only:
+    refs:
+      - master
+
+docker-build-dev:
+  image: docker:latest
+  stage: deploy
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - docker build --pull -t "$CI_REGISTRY_IMAGE/test:$CI_COMMIT_REF_SLUG" .
+    - docker push "$CI_REGISTRY_IMAGE/test:$CI_COMMIT_REF_SLUG"
+  except:
+    refs:
+      - master

Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:latest
+
+RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
+
+COPY rbxcompiler /bin/rbxcompiler

cmd/rbxcompiler/rbxcompiler.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"ci.itzana.me/itzaname/rbxcompiler/internal/rbxbuilder"
+	"fmt"
+)
+
+func main() {
+	/*fmt.Println(rbxbuilder.NewPlaceDump(&rbxbuilder.DumpSettings{
+		Source: "surf.rbxlx",
+		Output: "source",
+	}))*/
+
+	fmt.Println(rbxbuilder.NewPlaceBuilder(&rbxbuilder.BuildSettings{
+		Source: "source",
+		Output: "build.rbxlx",
+	}))
+}

go.mod

@@ -0,0 +1,8 @@
+module ci.itzana.me/itzaname/rbxcompiler
+
+go 1.15
+
+require (
+	ci.itzana.me/itzaname/rbxfile v0.0.0-20200929185118-23ef9783a53e
+	github.com/google/uuid v1.1.2
+)

go.sum

@@ -0,0 +1,8 @@
+ci.itzana.me/itzaname/rbxapi v0.1.0 h1:8tMoEvelXgxGJd71BXGBpGn/K18mWaWQvCsQqY7lnn4=
+ci.itzana.me/itzaname/rbxapi v0.1.0/go.mod h1:CRPbR/U4RqL4rqSGsEaYYr9wld3ctP+vClwgj/wGLsE=
+ci.itzana.me/itzaname/rbxfile v0.0.0-20200929185118-23ef9783a53e h1:xLqVw9gkdqKgywmEccJsXIJDM4bDgJ2g86ACe2iA7rA=
+ci.itzana.me/itzaname/rbxfile v0.0.0-20200929185118-23ef9783a53e/go.mod h1:YOKNgkzAvSDgoE795RTw/c/kzSxNzVvy12OkY/JcUVw=
+github.com/anaminus/but v0.2.0/go.mod h1:44z5qYo/3MWnZDi6ifH3IgrFWa1VFfdTttL3IYN/9R4=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

internal/rbxbuilder/build_place.go

@@ -0,0 +1,156 @@
+package rbxbuilder
+
+import (
+	"bytes"
+	"ci.itzana.me/itzaname/rbxfile"
+	"ci.itzana.me/itzaname/rbxfile/xml"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+type PlaceBuilder struct {
+	Options *BuildSettings
+	root    *rbxfile.Root
+	scripts map[string]string
+}
+
+func (b *PlaceBuilder) loadScript(path string) error {
+	scriptName := strings.TrimSuffix(filepath.Base(path), ".lua")
+	parentPath := filepath.Dir(path)
+	parentName := filepath.Base(parentPath)
+	basePath := strings.TrimSuffix(strings.TrimSuffix(path, parentName+string(os.PathSeparator)+filepath.Base(path)), seperator)
+	if basePath == "" {
+		basePath = parentPath
+	}
+
+	parentInstance := instanceFromScriptPath(b.root, parentPath)
+	parentBaseInstance := instanceFromScriptPath(b.root, basePath)
+	if parentBaseInstance == nil {
+		return fmt.Errorf("base instance doesn't exist: %s", basePath)
+	}
+	if parentInstance == nil {
+		class := "ModuleScript"
+		path := parentPath + seperator + parentName + ".lua"
+		if override, ok := b.scripts[path]; ok {
+			class = override
+		}
+		script := loadScript(parentName, class, b.Options.Source+path)
+		if script != nil {
+			if err := parentBaseInstance.AddChild(script); err != nil {
+				return err
+			}
+			parentInstance = script
+		} else {
+			if err := parentBaseInstance.AddChild(generateFolder(parentName)); err != nil {
+				return err
+			}
+			parentInstance = script
+		}
+	}
+
+	if scriptName == parentName {
+		return nil
+	}
+
+	// Load normal scripts
+	class := "ModuleScript"
+	if override, ok := b.scripts[path]; ok {
+		class = override
+	}
+	script := loadScript(scriptName, class, b.Options.Source+path)
+	if script != nil {
+		if err := parentInstance.AddChild(script); err != nil {
+			return err
+		}
+	} else {
+		return fmt.Errorf("failed to load script: %s", path)
+	}
+
+	return nil
+}
+
+func (b *PlaceBuilder) scriptTree() ([]string, error) {
+	var filelist []string
+	err := filepath.Walk(b.Options.Source, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if filepath.Ext(path) == ".lua" {
+				filelist = append(filelist, strings.TrimPrefix(path, b.Options.Source))
+			}
+		}
+		return nil
+	})
+
+	sort.Slice(filelist, func(i, j int) bool {
+		return strings.Count(filelist[i], string(os.PathSeparator)) < strings.Count(filelist[j], string(os.PathSeparator))
+	})
+
+	return filelist, err
+
+}
+
+func (b *PlaceBuilder) createScripts() error {
+	tree, err := b.scriptTree()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(tree); i++ {
+		if err := b.loadScript(tree[i]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *PlaceBuilder) loadManifest() error {
+	b.scripts = map[string]string{}
+	file, err := os.Open(b.Options.Source + seperator + "manifest.json")
+	if err != nil {
+		return fmt.Errorf("failed to load manifest file")
+	}
+	defer file.Close()
+
+	var manifest Manifest
+	if err := json.NewDecoder(file).Decode(&manifest); err != nil {
+		return fmt.Errorf("failed to load manifest file")
+	}
+
+	for i := 0; i < len(manifest.Override); i++ {
+		b.scripts[manifest.Override[i].Path] = manifest.Override[i].Class
+	}
+
+	buffer := bytes.NewBuffer(manifest.Template)
+	root, err := xml.Deserialize(buffer, nil)
+	if err != nil {
+		return fmt.Errorf("failed to load template from manifest")
+	}
+	b.root = root
+
+	return nil
+}
+
+func (b PlaceBuilder) Build() error {
+	if err := b.loadManifest(); err != nil {
+		return err
+	}
+
+	if err := b.createScripts(); err != nil {
+		return err
+	}
+
+	file, err := os.OpenFile(b.Options.Output, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open output file")
+	}
+	defer file.Close()
+
+	return xml.Serialize(file, nil, b.root)
+}

internal/rbxbuilder/dump_place.go

@@ -0,0 +1,159 @@
+package rbxbuilder
+
+import (
+	"bytes"
+	"ci.itzana.me/itzaname/rbxfile"
+	"ci.itzana.me/itzaname/rbxfile/xml"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+type PlaceDumper struct {
+	Options  *DumpSettings
+	source   *rbxfile.Root
+	template *rbxfile.Root
+	scripts  map[string]string
+}
+
+func (d *PlaceDumper) dumpScriptsFromInstance(instance *rbxfile.Instance) error {
+	for _, child := range instance.Children {
+		if isScript(child) {
+			path := getScriptFilePathString(child)
+			if child.ClassName != "ModuleScript" {
+				d.scripts[path] = child.ClassName
+			}
+			if err := os.MkdirAll(filepath.Dir(d.Options.Output+path), 0755); err != nil {
+				return err
+			}
+			if err := ioutil.WriteFile(d.Options.Output+path, []byte(getSource(child)), 0644); err != nil {
+				return err
+			}
+		}
+		if err := d.dumpScriptsFromInstance(child); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *PlaceDumper) findInstance(name string) *rbxfile.Instance {
+	for _, child := range d.template.Instances {
+		if child.Name() == name {
+			return child
+		}
+	}
+	return nil
+}
+
+func (d *PlaceDumper) removeScripts(instance *rbxfile.Instance) {
+	var queue []*rbxfile.Instance
+	for _, child := range instance.Children {
+		if isScript(child) {
+			queue = append(queue, child)
+			continue
+		}
+		d.removeScripts(child)
+	}
+
+	for i := 0; i < len(queue); i++ {
+		instance.RemoveChild(queue[i])
+	}
+}
+
+func (d *PlaceDumper) addAssets() {
+	for _, parent := range d.source.Instances {
+		for _, child := range parent.Children {
+			if !isScript(child) {
+				if destInst := d.findInstance(parent.Name()); destInst != nil {
+					newChild := child.Clone()
+					d.removeScripts(newChild)
+					destInst.AddChild(newChild)
+				}
+			}
+		}
+	}
+}
+
+func (d *PlaceDumper) createTemplate() {
+	d.template = rbxfile.NewRoot()
+
+	for _, child := range d.source.Instances {
+		newInst := child.Clone()
+		newInst.RemoveAll()
+		d.template.Instances = append(d.template.Instances, newInst)
+	}
+
+	d.addAssets()
+}
+
+func (d *PlaceDumper) loadRootFromFile() error {
+	file, err := os.Open(d.Options.Source)
+	if err != nil {
+		return err
+	}
+
+	root, err := xml.Deserialize(file, nil)
+	if err != nil {
+		return err
+	}
+	d.source = root
+
+	return nil
+}
+
+func (d *PlaceDumper) dumpScripts() error {
+	for i := 0; i < len(d.source.Instances); i++ {
+		if err := d.dumpScriptsFromInstance(d.source.Instances[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d PlaceDumper) writeManifest() error {
+	buffer := &bytes.Buffer{}
+	if err := xml.Serialize(buffer, nil, d.template); err != nil {
+		return err
+	}
+
+	manifest := Manifest{
+		Template: buffer.Bytes(),
+	}
+
+	for path, class := range d.scripts {
+		manifest.Override = append(manifest.Override, Script{
+			Path:  path,
+			Class: class,
+		})
+	}
+
+	file, err := os.OpenFile(d.Options.Output+seperator+"manifest.json", os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	encoder := json.NewEncoder(file)
+	encoder.SetIndent("", "\t")
+	return encoder.Encode(&manifest)
+}
+
+func (d PlaceDumper) Dump() error {
+	d.scripts = map[string]string{}
+	if err := d.loadRootFromFile(); err != nil {
+		return err
+	}
+	d.createTemplate()
+
+	if err := d.dumpScripts(); err != nil {
+		return err
+	}
+
+	if err := d.writeManifest(); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/rbxbuilder/rbxbuilder.go

@@ -0,0 +1,41 @@
+package rbxbuilder
+
+import "io"
+
+type DumpSettings struct {
+	Source   string
+	Output   string
+	Download bool
+	Asset    struct {
+		Id    int
+		Group int
+	}
+}
+
+type BuildSettings struct {
+	Source string
+	Output string
+	Writer *io.Writer
+}
+
+type Manifest struct {
+	Override []Script
+	Template []byte
+}
+
+type Script struct {
+	Path  string
+	Class string
+}
+
+func NewPlaceDump(options *DumpSettings) error {
+	return PlaceDumper{
+		Options: options,
+	}.Dump()
+}
+
+func NewPlaceBuilder(options *BuildSettings) error {
+	return PlaceBuilder{
+		Options: options,
+	}.Build()
+}

internal/rbxbuilder/util.go

@@ -0,0 +1,118 @@
+package rbxbuilder
+
+import (
+	"ci.itzana.me/itzaname/rbxfile"
+	"github.com/google/uuid"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+var seperator = string(os.PathSeparator)
+
+func isScript(instance *rbxfile.Instance) bool {
+	var scriptTypes = []string{"LocalScript", "ModuleScript", "Script"}
+	for i := 0; i < len(scriptTypes); i++ {
+		if instance.ClassName == scriptTypes[i] {
+			return true
+		}
+	}
+	return false
+
+}
+
+func isFolder(instance *rbxfile.Instance) bool {
+	return instance.ClassName == "Folder"
+}
+
+func getSource(instance *rbxfile.Instance) string {
+	if source, ok := instance.Properties["Source"]; ok {
+		return source.String()
+	}
+	return "BUILD ERROR: FAILED TO LOAD SOURCE"
+}
+
+func getPath(instance *rbxfile.Instance) []string {
+	if instance == nil {
+		return []string{}
+	}
+	return append(getPath(instance.Parent()), instance.Name())
+}
+
+func getPathString(instance *rbxfile.Instance) string {
+	return seperator + strings.Join(getPath(instance), seperator)
+}
+
+func getScriptFilePath(instance *rbxfile.Instance) []string {
+	path := getPath(instance)
+	if len(instance.Children) > 0 {
+		path = append(path, instance.Name())
+	}
+	if len(path) > 0 {
+		path[len(path)-1] += ".lua"
+	}
+	return path
+}
+
+func getScriptFilePathString(instance *rbxfile.Instance) string {
+	return seperator + strings.Join(getScriptFilePath(instance), seperator)
+}
+
+func instanceFromScriptPath(root *rbxfile.Root, path string) *rbxfile.Instance {
+	paths := strings.Split(strings.TrimPrefix(strings.TrimSuffix(path, ".lua"), seperator), seperator)
+	if len(path) == 0 {
+		return nil
+	}
+
+	for i := 0; i < len(root.Instances); i++ {
+		instance := root.Instances[i]
+		if instance.Name() != paths[0] {
+			continue
+		}
+		for j := 0; j < len(paths); j++ {
+			if instance.Name() == paths[j] {
+				if len(paths)-1 == j {
+					return instance
+				}
+			}
+			for _, child := range instance.Children {
+				if child.Name() == paths[j] {
+					if len(paths)-1 == j {
+						return child
+					}
+					instance = child
+					break
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func loadScript(name string, class string, path string) *rbxfile.Instance {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+
+	script := rbxfile.NewInstance(class, nil)
+	script.Properties["AttributesSerialize"] = rbxfile.NewValue(rbxfile.TypeBinaryString)
+	script.Properties["LinkedSource"] = rbxfile.NewValue(rbxfile.TypeContent)
+	script.Properties["Name"] = rbxfile.ValueString(name)
+	script.Properties["ScriptGuid"] = rbxfile.ValueString("{" + uuid.New().String() + "}")
+	script.Properties["Source"] = rbxfile.ValueProtectedString(data)
+	script.Properties["Tags"] = rbxfile.ValueBinaryString("")
+
+	return script
+}
+
+func generateFolder(name string) *rbxfile.Instance {
+	script := rbxfile.NewInstance("Folder", nil)
+	script.Properties["AttributesSerialize"] = rbxfile.NewValue(rbxfile.TypeBinaryString)
+	script.Properties["Name"] = rbxfile.ValueString(name)
+	script.Properties["SourceAssetId"] = rbxfile.ValueInt64(-1)
+	script.Properties["Tags"] = rbxfile.ValueBinaryString("")
+
+	return script
+}