From 44c589610c3b08ddde0ee41d78457b4e25265353 Mon Sep 17 00:00:00 2001 From: Carl Kittelberger Date: Mon, 5 Feb 2018 17:18:33 +0100 Subject: [PATCH] Initial commit. --- .gitignore | 76 +++++++++++++ backend/docker-compose.yml | 17 +++ backend/docker/images/convey/Dockerfile | 9 ++ backend/docker/images/convey/entrypoint.sh | 9 ++ backend/docker/images/couchdb/Dockerfile | 7 ++ backend/docker/images/couchdb/entrypoint.sh | 39 +++++++ backend/internal/database/database_server.go | 53 +++++++++ .../internal/database/log_deltas_database.go | 9 ++ backend/internal/database/logs_database.go | 9 ++ .../internal/database/migration_database.go | 55 ++++++++++ .../migrations/migration_0.0.0_to_0.0.1.go | 23 ++++ .../database/migrations/migration_test.go | 66 +++++++++++ .../database/migrations/registration.go | 103 ++++++++++++++++++ backend/internal/server.go | 4 + backend/main.go | 5 + 15 files changed, 484 insertions(+) create mode 100644 .gitignore create mode 100644 backend/docker-compose.yml create mode 100644 backend/docker/images/convey/Dockerfile create mode 100644 backend/docker/images/convey/entrypoint.sh create mode 100644 backend/docker/images/couchdb/Dockerfile create mode 100644 backend/docker/images/couchdb/entrypoint.sh create mode 100644 backend/internal/database/database_server.go create mode 100644 backend/internal/database/log_deltas_database.go create mode 100644 backend/internal/database/logs_database.go create mode 100644 backend/internal/database/migration_database.go create mode 100644 backend/internal/database/migrations/migration_0.0.0_to_0.0.1.go create mode 100644 backend/internal/database/migrations/migration_test.go create mode 100644 backend/internal/database/migrations/registration.go create mode 100644 backend/internal/server.go create mode 100644 backend/main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f813ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +############################################################################### +# GO + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test + +############################################################################### +# WINDOWS + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +############################################################################### +# JETBRAINS + +## Directory-based project format +.idea/ +# if you remove the above rule, at least ignore user-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# and these sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml + +## File-based project format +*.ipr +*.iws +*.iml + +## Additional for IntelliJ +out/ + +# generated by mpeltonen/sbt-idea plugin +.idea_modules/ + +# generated by JIRA plugin +atlassian-ide-plugin.xml + +# generated by Crashlytics plugin (for Android Studio and Intellij) +com_crashlytics_export_strings.xml diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml new file mode 100644 index 0000000..f6934c7 --- /dev/null +++ b/backend/docker-compose.yml @@ -0,0 +1,17 @@ +version: "3.4" + +services: + goconvey: + build: docker/images/convey + working_dir: /go/src/git.icedream.tech/icedream/loggalicious/backend + volumes: + - .:/go/src/git.icedream.tech/icedream/loggalicious/backend:ro + depends_on: + - couchdb + ports: + - 8123:8080 + + couchdb: + build: docker/images/couchdb + ports: + - 5984:5984 \ No newline at end of file diff --git a/backend/docker/images/convey/Dockerfile b/backend/docker/images/convey/Dockerfile new file mode 100644 index 0000000..cfe2143 --- /dev/null +++ b/backend/docker/images/convey/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.9-alpine3.7 + +RUN apk add --no-cache git alpine-sdk +RUN go get -v github.com/smartystreets/goconvey + +COPY entrypoint.sh /usr/local/bin/docker-convey-entrypoint +RUN chmod +x /usr/local/bin/docker-convey-entrypoint +ENTRYPOINT ["docker-convey-entrypoint"] + diff --git a/backend/docker/images/convey/entrypoint.sh b/backend/docker/images/convey/entrypoint.sh new file mode 100644 index 0000000..db904d1 --- /dev/null +++ b/backend/docker/images/convey/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh -ex + +sleep 5 + +echo "Working dir: $(pwd)" + +go get -d -v ./... + +exec goconvey -host 0.0.0.0 -launchBrowser false "$@" diff --git a/backend/docker/images/couchdb/Dockerfile b/backend/docker/images/couchdb/Dockerfile new file mode 100644 index 0000000..22afdb9 --- /dev/null +++ b/backend/docker/images/couchdb/Dockerfile @@ -0,0 +1,7 @@ +FROM couchdb:2 + +COPY entrypoint.sh /usr/local/bin/docker-couchdb-entrypoint +RUN chmod +x /usr/local/bin/* + +ENTRYPOINT ["docker-couchdb-entrypoint"] +CMD ["/opt/couchdb/bin/couchdb"] diff --git a/backend/docker/images/couchdb/entrypoint.sh b/backend/docker/images/couchdb/entrypoint.sh new file mode 100644 index 0000000..c65aac7 --- /dev/null +++ b/backend/docker/images/couchdb/entrypoint.sh @@ -0,0 +1,39 @@ +#!/bin/sh -e + +echo "Waiting for CouchDB to do first initial startup..." +/docker-entrypoint.sh "$@" >/dev/null 2>&1 & +pid=$! + +while ! curl --fail -s -o /dev/null http://127.0.0.1:5984; do + sleep 1 + printf "." +done + +echo " ok" + +createdb() { + curl --fail -s -o /dev/null -X GET "http://127.0.0.1:5984/$1" ||\ + curl --fail -s -o /dev/null -X PUT "http://127.0.0.1:5984/$1" +} + +echo "Ensuring initial databases exist..." + +createdb _global_changes +createdb _metadata +createdb _replicator +createdb _users + +kill "$pid" + +echo "Waiting for shutdown..." + +while curl --fail -o /dev/null -s http://127.0.0.1:5984; do + sleep 1 +done + +echo "" +echo "====================================" +echo "== ACTUAL COUCHDB LOG STARTS HERE ==" +echo "====================================" + +exec tini -- /docker-entrypoint.sh "$@" diff --git a/backend/internal/database/database_server.go b/backend/internal/database/database_server.go new file mode 100644 index 0000000..aa2d660 --- /dev/null +++ b/backend/internal/database/database_server.go @@ -0,0 +1,53 @@ +package database + +import ( + "net/http" + + couchdb "github.com/fjl/go-couchdb" +) + +const ( + dbName_Logs = "logs" + dbName_LogDeltas = "log_deltas" + dbName_Migration = "migration" +) + +type DatabaseServer struct { + *couchdb.Client +} + +func NewDatabaseServer(rawurl string, rt http.RoundTripper) (server *DatabaseServer, err error) { + couchClient, err := couchdb.NewClient(rawurl, rt) + if err != nil { + return + } + server = &DatabaseServer{ + Client: couchClient, + } + return +} + +func (s *DatabaseServer) Initialize() (err error) { + if _, err = s.Client.CreateDB(dbName_Logs); err != nil { + return + } + if _, err = s.Client.CreateDB(dbName_LogDeltas); err != nil { + return + } + if _, err = s.Client.CreateDB(dbName_Migration); err != nil { + return + } + return +} + +func (s *DatabaseServer) GetLogsDatabase() (db *LogsDatabase) { + return &LogsDatabase{DB: s.Client.DB(dbName_Logs)} +} + +func (s *DatabaseServer) GetLogDeltasDatabase() (db *LogDeltasDatabase) { + return &LogDeltasDatabase{DB: s.Client.DB(dbName_LogDeltas)} +} + +func (s *DatabaseServer) GetMigrationDatabase() (db *MigrationDatabase) { + return &MigrationDatabase{DB: s.Client.DB(dbName_Migration)} +} diff --git a/backend/internal/database/log_deltas_database.go b/backend/internal/database/log_deltas_database.go new file mode 100644 index 0000000..25bd69c --- /dev/null +++ b/backend/internal/database/log_deltas_database.go @@ -0,0 +1,9 @@ +package database + +import ( + "github.com/fjl/go-couchdb" +) + +type LogDeltasDatabase struct { + *couchdb.DB +} diff --git a/backend/internal/database/logs_database.go b/backend/internal/database/logs_database.go new file mode 100644 index 0000000..d76b0d5 --- /dev/null +++ b/backend/internal/database/logs_database.go @@ -0,0 +1,9 @@ +package database + +import ( + "github.com/fjl/go-couchdb" +) + +type LogsDatabase struct { + *couchdb.DB +} diff --git a/backend/internal/database/migration_database.go b/backend/internal/database/migration_database.go new file mode 100644 index 0000000..58f4a82 --- /dev/null +++ b/backend/internal/database/migration_database.go @@ -0,0 +1,55 @@ +package database + +import ( + "net/http" + + "github.com/blang/semver" + "github.com/fjl/go-couchdb" +) + +const ( + VersionDocumentId = "version" +) + +type MigrationVersionDocument struct { + Rev string `json:"_rev,omitempty"` + Version semver.Version +} + +type MigrationDatabase struct { + *couchdb.DB +} + +/* +GetVersion returns the current version as stored in the migration database. +If no document was found, defaults to returning "0.0.0" as semver.Version +object. +*/ +func (mdb *MigrationDatabase) GetVersion() (version semver.Version, rev string, err error) { + v := new(MigrationVersionDocument) + err = mdb.DB.Get(VersionDocumentId, v, nil) + if err != nil { + isActuallyAnError := true + if cerr, ok := err.(*couchdb.Error); ok { + // CouchDB error + if cerr.StatusCode == http.StatusNotFound { + isActuallyAnError = false + } + } + if isActuallyAnError { + return + } + err = nil + } else { + rev = v.Rev + version = v.Version + } + return +} + +func (mdb *MigrationDatabase) PutVersion(version semver.Version, rev string) (newrev string, err error) { + newrev, err = mdb.DB.Put(VersionDocumentId, MigrationVersionDocument{ + Version: version, + }, rev) + return +} diff --git a/backend/internal/database/migrations/migration_0.0.0_to_0.0.1.go b/backend/internal/database/migrations/migration_0.0.0_to_0.0.1.go new file mode 100644 index 0000000..d69392d --- /dev/null +++ b/backend/internal/database/migrations/migration_0.0.0_to_0.0.1.go @@ -0,0 +1,23 @@ +package migrations + +import ( + "git.icedream.tech/icedream/loggalicious/backend/internal/database" +) + +func init() { + // This MUST be here! NEVER EVER REMOVE THIS! + RegisterMigration("0.0.1", func(s *database.DatabaseServer) (err error) { + _, err = s.Client.CreateDB("migration") + return + }) + + RegisterMigration("0.0.1", func(s *database.DatabaseServer) (err error) { + if _, err = s.Client.CreateDB("logs"); err != nil { + return + } + if _, err = s.Client.CreateDB("log_deltas"); err != nil { + return + } + return + }) +} diff --git a/backend/internal/database/migrations/migration_test.go b/backend/internal/database/migrations/migration_test.go new file mode 100644 index 0000000..93f9d88 --- /dev/null +++ b/backend/internal/database/migrations/migration_test.go @@ -0,0 +1,66 @@ +package migrations_test + +import ( + "fmt" + "strings" + "testing" + + "git.icedream.tech/icedream/loggalicious/backend/internal/database" + "git.icedream.tech/icedream/loggalicious/backend/internal/database/migrations" + "github.com/fjl/go-couchdb" + . "github.com/smartystreets/goconvey/convey" +) + +const couchDBUrl = "http://couchdb:5984" + +func Test_Migration(t *testing.T) { + couchDB, err := couchdb.NewClient(couchDBUrl, nil) + if err != nil { + t.Error(err) + return + } + db := &database.DatabaseServer{Client: couchDB} + + // Wipe database + dbNames, err := couchDB.AllDBs() + if err != nil { + t.Error(err) + return + } + for _, dbName := range dbNames { + if strings.HasPrefix(dbName, "_") { + continue + } + err = couchDB.DeleteDB(dbName) + if err != nil { + t.Error(err) + return + } + } + + Convey("Migrations", t, func() { + + Convey("At least one version is registered", func() { + versions := migrations.RegisteredVersions() + + So(len(versions), ShouldBeGreaterThan, 0) + + for _, version := range versions { + Convey(fmt.Sprintf("Version %s should have at least one migration", version), func() { + So(len(migrations.RegisteredMigrations(version)), ShouldBeGreaterThan, 0) + }) + } + }) + + Convey("Migrations from blank setup works", func() { + oldVersion, _, err := db.GetMigrationDatabase().GetVersion() + So(err, ShouldBeNil) + err = migrations.Run(db) + So(err, ShouldBeNil) + version, _, err := db.GetMigrationDatabase().GetVersion() + So(err, ShouldBeNil) + So(version, ShouldNotEqual, oldVersion) + }) + + }) +} diff --git a/backend/internal/database/migrations/registration.go b/backend/internal/database/migrations/registration.go new file mode 100644 index 0000000..3db46f0 --- /dev/null +++ b/backend/internal/database/migrations/registration.go @@ -0,0 +1,103 @@ +package migrations + +import ( + "net/http" + + "git.icedream.tech/icedream/loggalicious/backend/internal/database" + "github.com/blang/semver" + "github.com/fjl/go-couchdb" +) + +type MigrationFunc func(*database.DatabaseServer) error + +var ( + orderedRegisteredVersions = []semver.Version{} + registeredMigrations = map[string][]MigrationFunc{} +) + +func RegisteredVersions() []semver.Version { + return orderedRegisteredVersions +} + +func RegisteredMigrations(version semver.Version) []MigrationFunc { + return registeredMigrations[version.String()] +} + +func RegisterVersionIfNotRegistered(version semver.Version) { + versionStr := version.String() + if _, ok := registeredMigrations[versionStr]; !ok { + registeredMigrations[versionStr] = []MigrationFunc{} + injected := false + for index, currentVersion := range orderedRegisteredVersions { + if currentVersion.GT(version) /* first version to be newer than this version */ { + newOrderedRegisteredVersions := append(orderedRegisteredVersions[0:index], version) + newOrderedRegisteredVersions = append(newOrderedRegisteredVersions, orderedRegisteredVersions[index:]...) + orderedRegisteredVersions = newOrderedRegisteredVersions + injected = true + break + } + } + if !injected { + orderedRegisteredVersions = append(orderedRegisteredVersions, version) + } + } +} + +func RegisterMigration(versionStr string, cb MigrationFunc) { + version := MustParseVersion(versionStr) + RegisterVersionIfNotRegistered(version) + + registeredMigrations[versionStr] = append(registeredMigrations[versionStr], cb) +} + +func MustParseVersion(version string) semver.Version { + semverVersion, err := semver.Parse(version) + if err != nil { + panic(err) + } + return semverVersion +} + +func Run(s *database.DatabaseServer) (err error) { + // Fetch current version + migrationDatabase := s.GetMigrationDatabase() + version, rev, err := migrationDatabase.GetVersion() + if err != nil { + isActuallyAnError := true + if cerr, ok := err.(*couchdb.Error); ok { + // CouchDB error + if cerr.StatusCode == http.StatusNotFound { + isActuallyAnError = false + } + } + if isActuallyAnError { + return + } + version = semver.Version{ + Major: 0, + Minor: 0, + Patch: 0, + } + err = nil + } + + for _, targetVersion := range orderedRegisteredVersions { + // log.Println("Check if migration is needed:", targetVersion) + if targetVersion.GT(version) { + // log.Println("Migrating database:", version, "->", targetVersion) + for _, cb := range registeredMigrations[targetVersion.String()] { + err = cb(s) + if err != nil { + return + } + } + version = targetVersion + rev, err = migrationDatabase.PutVersion(version, rev) + if err != nil { + return + } + } + } + + return +} diff --git a/backend/internal/server.go b/backend/internal/server.go new file mode 100644 index 0000000..8956310 --- /dev/null +++ b/backend/internal/server.go @@ -0,0 +1,4 @@ +package internal + +type Server struct { +} diff --git a/backend/main.go b/backend/main.go new file mode 100644 index 0000000..7905807 --- /dev/null +++ b/backend/main.go @@ -0,0 +1,5 @@ +package main + +func main() { + +}