Goreleaser (#241)

* add goreleaser tooling * add files + hook * update hooks * allow passing build-dir using cli args * build tweaks * tweak more * update drone and goreleaser * chill out tests * remove postgres * docker push on snapshot * update releaser
2024-12-22 05:44:08 +01:00 · 2021-09-24 13:14:20 +02:00 · 2021-09-24 13:14:20 +02:00 · d515c9f1ec
commit d515c9f1ec
parent 142f37f1bd
14 changed files with 253 additions and 136 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -8,8 +8,8 @@
 kind: pipeline
 type: docker
 name: default
-steps:

+steps:
 # We use golangci-lint for linting.
 # See: https://golangci-lint.run/
 - name: lint
@ -19,51 +19,72 @@ steps:
    path: /root/.cache/go-build
  - name: golangci-lint-cache
    path: /root/.cache/golangci-lint
+  - name: go-src
+    path: /go
  commands:
  - golangci-lint run --timeout 5m0s --tests=false --verbose
  when:
    event:
      include:
-        - pull_request
+      - pull_request

 - name: test
  image: golang:1.17.1-alpine3.14
  volumes:
  - name: go-build-cache
    path: /root/.cache/go-build
+  - name: go-src
+    path: /go
  commands:
-  - CGO_ENABLED=0 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -count 1 -p 1 ./...
-  - CGO_ENABLED=0 GTS_DB_TYPE="postgres" GTS_DB_ADDRESS="postgres" go test -count 1 -p 1 ./...
+  - CGO_ENABLED=0 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -p 1 ./...
  when:
    event:
      include:
-        - pull_request
+      - pull_request

- name: publish
-  image: plugins/docker
-  settings:
-    auto_tag: true
-    username: gotosocial
-    password:
-      from_secret: gts_docker_password
-    repo: superseriousbusiness/gotosocial
-    tags: latest
-  when:
-    event:
-      exclude:
-        - pull_request
-
-# We need a postgres service running for the test step.
-# See: https://docs.drone.io/pipeline/docker/syntax/services/
-services:
- name: postgres
-  image: postgres
+- name: snapshot
+  image: superseriousbusiness/gotosocial-drone-build:latest # https://github.com/superseriousbusiness/gotosocial-drone-build
+  volumes:
+  - name: go-build-cache
+    path: /root/.cache/go-build
+  - name: docker
+    path: /var/run/docker.sock
  environment:
-    POSTGRES_PASSWORD: postgres
+    DOCKER_USERNAME: gotosocial
+    DOCKER_PASSWORD:
+      from_secret: gts_docker_password
+  commands:
+  - /go/dockerlogin.sh
+  - goreleaser release --rm-dist --snapshot
+  - docker push superseriousbusiness/gotosocial:latest
  when:
    event:
      include:
-        - pull_request
+      - push
+    branch:
+      include:
+      - main
+
+- name: release
+  image: superseriousbusiness/gotosocial-drone-build:latest # https://github.com/superseriousbusiness/gotosocial-drone-build
+  volumes:
+  - name: go-build-cache
+    path: /root/.cache/go-build
+  - name: docker
+    path: /var/run/docker.sock
+  environment:
+    DOCKER_USERNAME: gotosocial
+    DOCKER_PASSWORD:
+      from_secret: gts_docker_password
+    GITHUB_TOKEN:
+      from_secret: github_token
+  commands:
+  - /go/dockerlogin.sh
+  - goreleaser release --rm-dist
+  when:
+    event:
+      include:
+      - tag

 # We can speed up builds significantly by caching build artifacts between runs.
 # See: https://docs.drone.io/pipeline/docker/syntax/volumes/host/
@ -74,6 +95,12 @@ volumes:
 - name: golangci-lint-cache
  host:
    path: /drone/gotosocial/golangci-lint
+- name: go-src
+  host:
+    path: /drone/gotosocial/go
+- name: docker
+  host:
+    path: /var/run/docker.sock

 trigger:
  repo:
@ -86,6 +113,6 @@ trigger:

 ---
 kind: signature
-hmac: 703dad12a9e92cbd415b23d82620608830a60a70168527118e2e9aab145f1099
+hmac: 8c39ebbac5e9cf4abde546a2b6b8b99a863804969474a5c8fc11f394f415e0ac

 ...
--- a/.gitignore
+++ b/.gitignore
@ -12,3 +12,9 @@ cp.out

 # exclude compiled mkdocs site
 site/
+
+# exclude compiled binaries
+dist/
+
+# exclude the copy of swagger.yaml moved into assets during packaging
+web/assets/swagger.yaml
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@ -0,0 +1,85 @@
+# https://goreleaser.com
+project_name: gotosocial
+before:
+  # https://goreleaser.com/customization/hooks/
+  hooks:
+  # tidy up and lint
+  - go mod tidy
+  - go fmt ./...
+  # generate the swagger.yaml file using go-swagger and bundle it into the assets directory
+  - swagger generate spec -o docs/api/swagger.yaml --scan-models
+  - sed -i "s/REPLACE_ME/{{ incpatch .Version }}/" docs/api/swagger.yaml
+  - cp docs/api/swagger.yaml web/assets/swagger.yaml
+  # install and bundle the web assets and styling
+  - yarn install --cwd web/gotosocial-styling
+  - node web/gotosocial-styling/index.js --build-dir="web/assets"
+builds:
+  # https://goreleaser.com/customization/build/
+  -
+    main: ./cmd/gotosocial
+    binary: gotosocial
+    ldflags:
+      - -s
+      - -w
+      - -extldflags
+      - -static
+      - -X main.Commit={{.Commit}}
+      - -X main.Version={{.Version}}
+    tags:
+      - netgo
+      - osusergo
+      - static_build
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - freebsd
+    goarch:
+      - 386
+      - amd64
+      - arm
+      - arm64
+    ignore:
+      # build freebsd only for amd64
+      - goos: freebsd
+        goarch: arm64
+      - goos: freebsd
+        goarch: arm
+      - goos: freebsd
+        goarch: 386
+    mod_timestamp: "{{ .CommitTimestamp }}"
+dockers:
+  # https://goreleaser.com/customization/docker/
+  -
+    goos: linux
+    goarch: amd64
+    image_templates:
+    - "superseriousbusiness/gotosocial:latest"
+    - "superseriousbusiness/gotosocial:{{ .Version }}"
+    build_flag_templates:
+    - "--label=org.opencontainers.image.created={{.Date}}"
+    - "--label=org.opencontainers.image.title={{.ProjectName}}"
+    - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+    - "--label=org.opencontainers.image.version={{.Version}}"
+    extra_files:
+    - web
+archives:
+  # https://goreleaser.com/customization/archive/
+  -
+    files:
+    # standard release files
+    - LICENSE
+    - README.md
+    - CHANGELOG*
+    # web assets and example config
+    - web
+    - example/config.yaml
+checksum:
+  # https://goreleaser.com/customization/checksum/
+  name_template: 'checksums.txt'
+snapshot:
+  # https://goreleaser.com/customization/snapshots/
+  name_template: "{{ incpatch .Version }}-SNAPSHOT"
+source:
+  # https://goreleaser.com/customization/source/
+  enabled: true
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -13,17 +13,20 @@ Check the [issues](https://github.com/superseriousbusiness/gotosocial/issues) to
 - [Communications](#communications)
 - [Code of Conduct](#code-of-conduct)
 - [Setting up your development environment](#setting-up-your-development-environment)
+  - [Stylesheet / Web dev](#stylesheet--web-dev)
  - [Golang forking quirks](#golang-forking-quirks)
 - [Setting up your test environment](#setting-up-your-test-environment)
-  - [Standalone Testrig](#standalone-testrig)
- [Running tests](#running-tests)
-  - [SQLite](#sqlite)
-  - [Postgres](#postgres)
-  - [Both](#both)
+  - [Standalone Testrig with Pinafore](#standalone-testrig-with-pinafore)
+  - [Running automated tests](#running-automated-tests)
+    - [SQLite](#sqlite)
+    - [Postgres](#postgres)
+    - [Both](#both)
 - [Linting](#linting)
 - [Updating Swagger docs](#updating-swagger-docs)
- [Pushing to Docker](#pushing-to-docker)
 - [CI/CD configuration](#cicd-configuration)
+- [Building releases and Docker containers](#building-releases-and-docker-containers)
+  - [With GoReleaser](#with-goreleaser)
+  - [Manually](#manually)
 - [Financial Compensation](#financial-compensation)

 ## Communications
@ -46,11 +49,37 @@ To get started, you first need to have Go installed. GtS is currently using Go 1

 Once you've got go installed, clone this repository into your Go path. Normally, this should be `~/go/src/github.com/superseriousbusiness/gotosocial`.

-Once that's done, you can try building the project: `./scripts/build.sh`. This will build the `gotosocial` binary. For automatic re-compiling during development, you can use [nodemon](https://www.npmjs.com/package/nodemon): `nodemon -e go --signal SIGTERM --exec "go run ./cmd/gotosocial --host localhost testrig start || exit 1"`
+Once that's done, you can try building the project: `./scripts/build.sh`. This will build the `gotosocial` binary.

 If there are no errors, great, you're good to go!

-To work with the stylesheet for templates, you need [Node.js](https://nodejs.org/en/download/), then run `yarn install` in `web/gotosocial-styling/`. Recompiling the bundles is done with `BUILD_DIR=../assets node index.js` but can be automatically live-reloaded with `BUILD_DIR=../assets NODE_ENV=development node index.js`. 
+For automatic re-compiling during development, you can use [nodemon](https://www.npmjs.com/package/nodemon):
+
+```bash
+nodemon -e go --signal SIGTERM --exec "go run ./cmd/gotosocial --host localhost testrig start || exit 1"
+```
+
+### Stylesheet / Web dev
+
+To work with the stylesheet for templates, you need [Node.js](https://nodejs.org/en/download/) and [Yarn](https://classic.yarnpkg.com/en/docs/install).
+
+To install Yarn dependencies:
+
+```bash
+yarn install --cwd web/gotosocial-styling
+```
+
+To recompile bundles:
+
+```bash
+node web/gotosocial-styling/index.js --build-dir="web/assets"
+```
+
+You can do automatic live-reloads of bundles with:
+
+``` bash
+NODE_ENV=development node web/gotosocial-styling/index.js --build-dir="web/assets"
+```

 ### Golang forking quirks

@ -90,9 +119,9 @@ GoToSocial provides a [testrig](https://github.com/superseriousbusiness/gotosoci

 One thing that *isn't* mocked is the Database interface, because it's just easier to use an in-memory SQLite database than to mock everything out.

-### Standalone Testrig
+### Standalone Testrig with Pinafore

-You can also launch a testrig as a standalone server running at localhost, which you can connect to using something like [Pinafore](https://github.com/nolanlawson/pinafore).
+You can launch a testrig as a standalone server running at localhost, which you can connect to using something like [Pinafore](https://github.com/nolanlawson/pinafore).

 To do this, first build the gotosocial binary with `./scripts/build.sh`.

@ -102,7 +131,7 @@ Then, launch the testrig by invoking the binary as follows:
 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" ./gotosocial --host localhost:8080 testrig start
 ```

-To run Pinafore locally in dev mode, first clone the Pinafore repository, and run the following command in the cloned directory:
+To run Pinafore locally in dev mode, first clone the [Pinafore](https://github.com/nolanlawson/pinafore) repository, and then run the following command in the cloned directory:

 ```bash
 yarn run dev
@ -112,19 +141,19 @@ The Pinafore instance will start running on `localhost:4002`.

 To connect to the testrig, navigate to `https://localhost:4002` and enter your instance name as `localhost:8080`.

-At the login screen, enter the email address `zork@example.org` and password `password`.
+At the login screen, enter the email address `zork@example.org` and password `password`. You will get a confirmation prompt. Accept, and you are logged in as Zork.

 Note the following constraints:

 - Since the testrig uses an in-memory database, the database will be destroyed when the testrig is stopped.
 - If you stop the testrig and start it again, any tokens or applications you created during your tests will also be removed. As such, you need to log out and in again every time you stop/start the rig.
- The testrig does not make any actual external http calls, so federation will (obviously) not work from a testrig.
+- The testrig does not make any actual external http calls, so federation will not work from a testrig.

-## Running tests
+### Running automated tests

 There are a few different ways of running tests. Each requires the use of the `-p 1` flag, to indicate that they should not be run in parallel.

-### SQLite
+#### SQLite

 If you want to run tests as quickly as possible, using an SQLite in-memory database, use:

@ -132,7 +161,7 @@ If you want to run tests as quickly as possible, using an SQLite in-memory datab
 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -p 1 ./...
 ```

-### Postgres
+#### Postgres

 If you want to run tests against a Postgres database running on localhost, run:

@ -142,7 +171,7 @@ GTS_DB_TYPE="postgres" GTS_DB_ADDRESS="localhost" go test -p 1 ./...

 In the above command, it is assumed you are using the default Postgres password of `postgres`.

-### Both
+#### Both

 Finally, to run tests against both database types one after the other, use:

@ -180,29 +209,19 @@ Then make sure to run `go fmt ./...` to update whitespace and other opinionated

 ## Updating Swagger docs

-If you change swagger annotations on any of the API paths, you need to generate a new swagger file at `./docs/api/swagger.yaml`. You can do this with:
+GoToSocial uses [go-swagger](https://goswagger.io) to generate Swagger API documentation from code annotations.

-`./scripts/generateswagger.sh`
+You can install go-swagger following the instructions [here](https://goswagger.io/install.html).

-## Pushing to Docker
+If you change Swagger annotations on any of the API paths, you can generate a new Swagger file at `./docs/api/swagger.yaml` by running:

-You can easily build a Docker container tagged with the current branch name using:
-
-```bash
-./scripts/dockerbuild.sh
-```
-
-Then, (assuming you have permissions to push to the [GoToSocial Docker repository](https://hub.docker.com/r/superseriousbusiness/gotosocial)), run:
-
-```bash
-./scripts/dockerpush.sh
-```
-
-Note: you should never manually push a Docker container from your machine to `latest` -- we have a CI/CD flow for that. Only push a Docker container manually when you're testing changes on a branch!
+`swagger generate spec -o docs/api/swagger.yaml --scan-models`

 ## CI/CD configuration

-GoToSocial uses [Drone](https://www.drone.io/) for CI/CD tasks like running tests, linting, and building Docker containers. These runs are integrated with Github, and will be run on opening a pull request or merging into main.
+GoToSocial uses [Drone](https://www.drone.io/) for CI/CD tasks like running tests, linting, and building Docker containers.
+
+These runs are integrated with Github, and will be run on opening a pull request or merging into main.

 The Drone instance for GoToSocial is [here](https://drone.superseriousbusiness.org/superseriousbusiness/gotosocial).

@ -216,6 +235,34 @@ To sign the file, first install and setup the [drone cli tool](https://docs.dron
 drone -t PUT_YOUR_DRONE_ADMIN_TOKEN_HERE -s https://drone.superseriousbusiness.org sign superseriousbusiness/gotosocial --save
 ```

+## Building releases and Docker containers
+
+### With GoReleaser
+
+GoToSocial uses the release tooling [GoReleaser](https://goreleaser.com/intro/) to make multiple-architecture + Docker builds simple.
+
+GoReleaser is also used by GoToSocial for building and pushing Docker containers.
+
+Normally, these processes are handled by Drone (see CI/CD above). However, you can also invoke GoReleaser manually for things like building snapshots.
+
+To do this, first [install GoReleaser](https://goreleaser.com/install/).
+
+Then, to create snapshot builds, do:
+
+```bash
+goreleaser release --rm-dist --snapshot
+```
+
+If all goes according to plan, you should now have a bunch of multiple-architecture binaries and tars inside the `./dist` folder, and a snapshot Docker image should be built (check your terminal output for version).
+
+### Manually
+
+If you prefer a simple approach with fewer dependencies, you can also just build a Docker container manually in the following way:
+
+```bash
+./scripts/build.sh && docker build -t superseriousbusiness/gotosocial:latest .
+```
+
 ## Financial Compensation

 Right now there's no structure in place for financial compensation for pull requests and code. This is simply because there's no money being made on the project apart from the very small weekly Liberapay donations.
--- a/50
+++ b/50
@ -1,44 +1,4 @@
-# STEP ONE: build the GoToSocial binary
-FROM golang:1.17.1-alpine3.14 AS binary_builder
-RUN apk update && apk upgrade --no-cache
-RUN apk add git
-
-# create build dir
-RUN mkdir -p /go/src/github.com/superseriousbusiness/gotosocial
-WORKDIR /go/src/github.com/superseriousbusiness/gotosocial
-
-# move source files
-ADD cmd /go/src/github.com/superseriousbusiness/gotosocial/cmd
-ADD internal /go/src/github.com/superseriousbusiness/gotosocial/internal
-ADD testrig /go/src/github.com/superseriousbusiness/gotosocial/testrig
-ADD docs/swagger.go /go/src/github.com/superseriousbusiness/gotosocial/docs/swagger.go
-
-# dependencies and vendor
-ADD go.mod /go/src/github.com/superseriousbusiness/gotosocial/go.mod
-ADD go.sum /go/src/github.com/superseriousbusiness/gotosocial/go.sum
-ADD vendor /go/src/github.com/superseriousbusiness/gotosocial/vendor
-
-# move .git dir and version for versioning
-ADD .git /go/src/github.com/superseriousbusiness/gotosocial/.git
-ADD version /go/src/github.com/superseriousbusiness/gotosocial/version
-
-# move the build script
-ADD scripts/build.sh /go/src/github.com/superseriousbusiness/gotosocial/build.sh
-
-# do the build step
-RUN ./build.sh
-
-# STEP TWO: build the web assets
-FROM node:16.9.0-alpine3.14 AS web_builder
-RUN apk update && apk upgrade --no-cache
-
-COPY web /web
-WORKDIR /web/gotosocial-styling
-
-RUN yarn install
-RUN BUILD_DIR=../assets node index.js
-
-# STEP THREE: bundle the admin webapp
+# bundle the admin webapp
 FROM node:16.9.0-alpine3.14 AS admin_builder
 RUN apk update && apk upgrade --no-cache
 RUN apk add git
@ -49,19 +9,15 @@ WORKDIR /gotosocial-admin
 RUN npm install
 RUN node index.js

-# STEP FOUR: build the final container
 FROM alpine:3.14.2 AS executor
 RUN apk update && apk upgrade --no-cache

 # copy over the binary from the first stage
 RUN mkdir -p /gotosocial/storage
-COPY --from=binary_builder /go/src/github.com/superseriousbusiness/gotosocial/gotosocial /gotosocial/gotosocial
+COPY gotosocial /gotosocial/gotosocial

 # copy over the web directory with templates etc
-COPY --from=web_builder web /gotosocial/web
-
-# put the swagger yaml in the web assets directory so it can be accessed
-COPY docs/api/swagger.yaml /gotosocial/web/assets/swagger.yaml
+COPY web /gotosocial/web

 # copy over the admin directory
 COPY --from=admin_builder /gotosocial-admin/public /gotosocial/web/assets/admin
--- a/docs/api/swagger.yaml
+++ b/docs/api/swagger.yaml
@ -1679,7 +1679,7 @@ info:
    name: AGPL3
    url: https://www.gnu.org/licenses/agpl-3.0.en.html
  title: GoToSocial
-  version: 0.1.0-SNAPSHOT
+  version: 0.0.1
 paths:
  /api/v1/accounts:
    post:
--- a/scripts/build.sh
+++ b/scripts/build.sh
@ -2,8 +2,8 @@

 set -eu

-COMMIT=$(git rev-list -1 HEAD)
-VERSION=$(cat ./version)
+COMMIT="${COMMIT:-12345678}"
+VERSION="${VERSION:-0.0.0}"

 CGO_ENABLED=0 go build -trimpath \
                       -tags 'netgo osusergo static_build' \
--- a/scripts/dockerbuild.sh
+++ b/scripts/dockerbuild.sh
@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)"
-
-docker build -t "superseriousbusiness/gotosocial:${BRANCH_NAME}" .
--- a/scripts/dockerpush.sh
+++ b/scripts/dockerpush.sh
@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -e
-
-BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)"
-
-docker push "superseriousbusiness/gotosocial:${BRANCH_NAME}"
--- a/scripts/generateswagger.sh
+++ b/scripts/generateswagger.sh
@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-SWAGGER_FILE="docs/api/swagger.yaml"
-GTS_VERSION="$(cat version)"
-
-swagger generate spec -o "${SWAGGER_FILE}" --scan-models
-sed -i "s/REPLACE_ME/${GTS_VERSION}/" "${SWAGGER_FILE}"
--- a/1
+++ b/1
@ -1 +0,0 @@
-0.1.0-SNAPSHOT
--- a/web/gotosocial-styling/index.js
+++ b/web/gotosocial-styling/index.js
@ -4,6 +4,7 @@ const Promise = require("bluebird");
 const fs = require("fs").promises;
 const postcss = require('postcss');
 const {parse} = require("postcss-scss");
+const argv = require('minimist')(process.argv.slice(2));

 /*
 	Bundle all postCSS files under the `templates/` directory separately, each prepended with the (variable) contents of ./colors.css
@ -42,10 +43,23 @@ function bundle([template, path]) {
 	});
 }

-let buildDir = process.env.BUILD_DIR;
+let buildDir
+
+// try reading from arguments first
+if (argv["build-dir"] != undefined) {
+	buildDir = argv["build-dir"]
+}
+
+// then try reading from environment variable
+if (buildDir == undefined) {
+	buildDir = process.env.BUILD_DIR;
+}
+
+// then take default
 if (buildDir == undefined) {
 	buildDir = `${__dirname}/build`;
 }
+
 console.log("bundling to", buildDir);

 function bundleAll() {
--- a/web/gotosocial-styling/package.json
+++ b/web/gotosocial-styling/package.json
@ -7,6 +7,7 @@
  "license": "AGPL-3.0",
  "dependencies": {
    "bluebird": "^3.7.2",
+    "minimist": "^1.2.5",
    "postcss": "^8.3.5",
    "postcss-color-function": "^4.1.0",
    "postcss-nested": "^4.2.1",
--- a/web/gotosocial-styling/yarn.lock
+++ b/web/gotosocial-styling/yarn.lock
@ -219,6 +219,11 @@ js-base64@^2.1.9:
  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.6.4.tgz#f4e686c5de1ea1f867dbcad3d46d969428df98c4"
  integrity sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==

+minimist@^1.2.5:
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+
 ms@^2.1.1:
  version "2.1.3"
  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"