diff --git a/.drone.yml b/.drone.yml index d8af6a83a..8111c82ad 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,8 +8,8 @@ kind: pipeline type: docker name: default -steps: +steps: # We use golangci-lint for linting. # See: https://golangci-lint.run/ - name: lint @@ -19,51 +19,72 @@ steps: path: /root/.cache/go-build - name: golangci-lint-cache path: /root/.cache/golangci-lint + - name: go-src + path: /go commands: - golangci-lint run --timeout 5m0s --tests=false --verbose when: event: include: - - pull_request + - pull_request - name: test image: golang:1.17.1-alpine3.14 volumes: - name: go-build-cache path: /root/.cache/go-build + - name: go-src + path: /go commands: - - CGO_ENABLED=0 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -count 1 -p 1 ./... - - CGO_ENABLED=0 GTS_DB_TYPE="postgres" GTS_DB_ADDRESS="postgres" go test -count 1 -p 1 ./... + - CGO_ENABLED=0 GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -p 1 ./... when: event: include: - - pull_request + - pull_request -- name: publish - image: plugins/docker - settings: - auto_tag: true - username: gotosocial - password: - from_secret: gts_docker_password - repo: superseriousbusiness/gotosocial - tags: latest - when: - event: - exclude: - - pull_request - -# We need a postgres service running for the test step. -# See: https://docs.drone.io/pipeline/docker/syntax/services/ -services: -- name: postgres - image: postgres +- name: snapshot + image: superseriousbusiness/gotosocial-drone-build:latest # https://github.com/superseriousbusiness/gotosocial-drone-build + volumes: + - name: go-build-cache + path: /root/.cache/go-build + - name: docker + path: /var/run/docker.sock environment: - POSTGRES_PASSWORD: postgres + DOCKER_USERNAME: gotosocial + DOCKER_PASSWORD: + from_secret: gts_docker_password + commands: + - /go/dockerlogin.sh + - goreleaser release --rm-dist --snapshot + - docker push superseriousbusiness/gotosocial:latest when: event: include: - - pull_request + - push + branch: + include: + - main + +- name: release + image: superseriousbusiness/gotosocial-drone-build:latest # https://github.com/superseriousbusiness/gotosocial-drone-build + volumes: + - name: go-build-cache + path: /root/.cache/go-build + - name: docker + path: /var/run/docker.sock + environment: + DOCKER_USERNAME: gotosocial + DOCKER_PASSWORD: + from_secret: gts_docker_password + GITHUB_TOKEN: + from_secret: github_token + commands: + - /go/dockerlogin.sh + - goreleaser release --rm-dist + when: + event: + include: + - tag # We can speed up builds significantly by caching build artifacts between runs. # See: https://docs.drone.io/pipeline/docker/syntax/volumes/host/ @@ -74,6 +95,12 @@ volumes: - name: golangci-lint-cache host: path: /drone/gotosocial/golangci-lint +- name: go-src + host: + path: /drone/gotosocial/go +- name: docker + host: + path: /var/run/docker.sock trigger: repo: @@ -86,6 +113,6 @@ trigger: --- kind: signature -hmac: 703dad12a9e92cbd415b23d82620608830a60a70168527118e2e9aab145f1099 +hmac: 8c39ebbac5e9cf4abde546a2b6b8b99a863804969474a5c8fc11f394f415e0ac ... diff --git a/.gitignore b/.gitignore index 7f1620871..536dc0c00 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ cp.out # exclude compiled mkdocs site site/ + +# exclude compiled binaries +dist/ + +# exclude the copy of swagger.yaml moved into assets during packaging +web/assets/swagger.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 000000000..0f39d3186 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,85 @@ +# https://goreleaser.com +project_name: gotosocial +before: + # https://goreleaser.com/customization/hooks/ + hooks: + # tidy up and lint + - go mod tidy + - go fmt ./... + # generate the swagger.yaml file using go-swagger and bundle it into the assets directory + - swagger generate spec -o docs/api/swagger.yaml --scan-models + - sed -i "s/REPLACE_ME/{{ incpatch .Version }}/" docs/api/swagger.yaml + - cp docs/api/swagger.yaml web/assets/swagger.yaml + # install and bundle the web assets and styling + - yarn install --cwd web/gotosocial-styling + - node web/gotosocial-styling/index.js --build-dir="web/assets" +builds: + # https://goreleaser.com/customization/build/ + - + main: ./cmd/gotosocial + binary: gotosocial + ldflags: + - -s + - -w + - -extldflags + - -static + - -X main.Commit={{.Commit}} + - -X main.Version={{.Version}} + tags: + - netgo + - osusergo + - static_build + env: + - CGO_ENABLED=0 + goos: + - linux + - freebsd + goarch: + - 386 + - amd64 + - arm + - arm64 + ignore: + # build freebsd only for amd64 + - goos: freebsd + goarch: arm64 + - goos: freebsd + goarch: arm + - goos: freebsd + goarch: 386 + mod_timestamp: "{{ .CommitTimestamp }}" +dockers: + # https://goreleaser.com/customization/docker/ + - + goos: linux + goarch: amd64 + image_templates: + - "superseriousbusiness/gotosocial:latest" + - "superseriousbusiness/gotosocial:{{ .Version }}" + build_flag_templates: + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + extra_files: + - web +archives: + # https://goreleaser.com/customization/archive/ + - + files: + # standard release files + - LICENSE + - README.md + - CHANGELOG* + # web assets and example config + - web + - example/config.yaml +checksum: + # https://goreleaser.com/customization/checksum/ + name_template: 'checksums.txt' +snapshot: + # https://goreleaser.com/customization/snapshots/ + name_template: "{{ incpatch .Version }}-SNAPSHOT" +source: + # https://goreleaser.com/customization/source/ + enabled: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 253a40310..d6a37e418 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,17 +13,20 @@ Check the [issues](https://github.com/superseriousbusiness/gotosocial/issues) to - [Communications](#communications) - [Code of Conduct](#code-of-conduct) - [Setting up your development environment](#setting-up-your-development-environment) + - [Stylesheet / Web dev](#stylesheet--web-dev) - [Golang forking quirks](#golang-forking-quirks) - [Setting up your test environment](#setting-up-your-test-environment) - - [Standalone Testrig](#standalone-testrig) -- [Running tests](#running-tests) - - [SQLite](#sqlite) - - [Postgres](#postgres) - - [Both](#both) + - [Standalone Testrig with Pinafore](#standalone-testrig-with-pinafore) + - [Running automated tests](#running-automated-tests) + - [SQLite](#sqlite) + - [Postgres](#postgres) + - [Both](#both) - [Linting](#linting) - [Updating Swagger docs](#updating-swagger-docs) -- [Pushing to Docker](#pushing-to-docker) - [CI/CD configuration](#cicd-configuration) +- [Building releases and Docker containers](#building-releases-and-docker-containers) + - [With GoReleaser](#with-goreleaser) + - [Manually](#manually) - [Financial Compensation](#financial-compensation) ## Communications @@ -46,11 +49,37 @@ To get started, you first need to have Go installed. GtS is currently using Go 1 Once you've got go installed, clone this repository into your Go path. Normally, this should be `~/go/src/github.com/superseriousbusiness/gotosocial`. -Once that's done, you can try building the project: `./scripts/build.sh`. This will build the `gotosocial` binary. For automatic re-compiling during development, you can use [nodemon](https://www.npmjs.com/package/nodemon): `nodemon -e go --signal SIGTERM --exec "go run ./cmd/gotosocial --host localhost testrig start || exit 1"` +Once that's done, you can try building the project: `./scripts/build.sh`. This will build the `gotosocial` binary. If there are no errors, great, you're good to go! -To work with the stylesheet for templates, you need [Node.js](https://nodejs.org/en/download/), then run `yarn install` in `web/gotosocial-styling/`. Recompiling the bundles is done with `BUILD_DIR=../assets node index.js` but can be automatically live-reloaded with `BUILD_DIR=../assets NODE_ENV=development node index.js`. +For automatic re-compiling during development, you can use [nodemon](https://www.npmjs.com/package/nodemon): + +```bash +nodemon -e go --signal SIGTERM --exec "go run ./cmd/gotosocial --host localhost testrig start || exit 1" +``` + +### Stylesheet / Web dev + +To work with the stylesheet for templates, you need [Node.js](https://nodejs.org/en/download/) and [Yarn](https://classic.yarnpkg.com/en/docs/install). + +To install Yarn dependencies: + +```bash +yarn install --cwd web/gotosocial-styling +``` + +To recompile bundles: + +```bash +node web/gotosocial-styling/index.js --build-dir="web/assets" +``` + +You can do automatic live-reloads of bundles with: + +``` bash +NODE_ENV=development node web/gotosocial-styling/index.js --build-dir="web/assets" +``` ### Golang forking quirks @@ -90,9 +119,9 @@ GoToSocial provides a [testrig](https://github.com/superseriousbusiness/gotosoci One thing that *isn't* mocked is the Database interface, because it's just easier to use an in-memory SQLite database than to mock everything out. -### Standalone Testrig +### Standalone Testrig with Pinafore -You can also launch a testrig as a standalone server running at localhost, which you can connect to using something like [Pinafore](https://github.com/nolanlawson/pinafore). +You can launch a testrig as a standalone server running at localhost, which you can connect to using something like [Pinafore](https://github.com/nolanlawson/pinafore). To do this, first build the gotosocial binary with `./scripts/build.sh`. @@ -102,7 +131,7 @@ Then, launch the testrig by invoking the binary as follows: GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" ./gotosocial --host localhost:8080 testrig start ``` -To run Pinafore locally in dev mode, first clone the Pinafore repository, and run the following command in the cloned directory: +To run Pinafore locally in dev mode, first clone the [Pinafore](https://github.com/nolanlawson/pinafore) repository, and then run the following command in the cloned directory: ```bash yarn run dev @@ -112,19 +141,19 @@ The Pinafore instance will start running on `localhost:4002`. To connect to the testrig, navigate to `https://localhost:4002` and enter your instance name as `localhost:8080`. -At the login screen, enter the email address `zork@example.org` and password `password`. +At the login screen, enter the email address `zork@example.org` and password `password`. You will get a confirmation prompt. Accept, and you are logged in as Zork. Note the following constraints: - Since the testrig uses an in-memory database, the database will be destroyed when the testrig is stopped. - If you stop the testrig and start it again, any tokens or applications you created during your tests will also be removed. As such, you need to log out and in again every time you stop/start the rig. -- The testrig does not make any actual external http calls, so federation will (obviously) not work from a testrig. +- The testrig does not make any actual external http calls, so federation will not work from a testrig. -## Running tests +### Running automated tests There are a few different ways of running tests. Each requires the use of the `-p 1` flag, to indicate that they should not be run in parallel. -### SQLite +#### SQLite If you want to run tests as quickly as possible, using an SQLite in-memory database, use: @@ -132,7 +161,7 @@ If you want to run tests as quickly as possible, using an SQLite in-memory datab GTS_DB_TYPE="sqlite" GTS_DB_ADDRESS=":memory:" go test -p 1 ./... ``` -### Postgres +#### Postgres If you want to run tests against a Postgres database running on localhost, run: @@ -142,7 +171,7 @@ GTS_DB_TYPE="postgres" GTS_DB_ADDRESS="localhost" go test -p 1 ./... In the above command, it is assumed you are using the default Postgres password of `postgres`. -### Both +#### Both Finally, to run tests against both database types one after the other, use: @@ -180,29 +209,19 @@ Then make sure to run `go fmt ./...` to update whitespace and other opinionated ## Updating Swagger docs -If you change swagger annotations on any of the API paths, you need to generate a new swagger file at `./docs/api/swagger.yaml`. You can do this with: +GoToSocial uses [go-swagger](https://goswagger.io) to generate Swagger API documentation from code annotations. -`./scripts/generateswagger.sh` +You can install go-swagger following the instructions [here](https://goswagger.io/install.html). -## Pushing to Docker +If you change Swagger annotations on any of the API paths, you can generate a new Swagger file at `./docs/api/swagger.yaml` by running: -You can easily build a Docker container tagged with the current branch name using: - -```bash -./scripts/dockerbuild.sh -``` - -Then, (assuming you have permissions to push to the [GoToSocial Docker repository](https://hub.docker.com/r/superseriousbusiness/gotosocial)), run: - -```bash -./scripts/dockerpush.sh -``` - -Note: you should never manually push a Docker container from your machine to `latest` -- we have a CI/CD flow for that. Only push a Docker container manually when you're testing changes on a branch! +`swagger generate spec -o docs/api/swagger.yaml --scan-models` ## CI/CD configuration -GoToSocial uses [Drone](https://www.drone.io/) for CI/CD tasks like running tests, linting, and building Docker containers. These runs are integrated with Github, and will be run on opening a pull request or merging into main. +GoToSocial uses [Drone](https://www.drone.io/) for CI/CD tasks like running tests, linting, and building Docker containers. + +These runs are integrated with Github, and will be run on opening a pull request or merging into main. The Drone instance for GoToSocial is [here](https://drone.superseriousbusiness.org/superseriousbusiness/gotosocial). @@ -216,6 +235,34 @@ To sign the file, first install and setup the [drone cli tool](https://docs.dron drone -t PUT_YOUR_DRONE_ADMIN_TOKEN_HERE -s https://drone.superseriousbusiness.org sign superseriousbusiness/gotosocial --save ``` +## Building releases and Docker containers + +### With GoReleaser + +GoToSocial uses the release tooling [GoReleaser](https://goreleaser.com/intro/) to make multiple-architecture + Docker builds simple. + +GoReleaser is also used by GoToSocial for building and pushing Docker containers. + +Normally, these processes are handled by Drone (see CI/CD above). However, you can also invoke GoReleaser manually for things like building snapshots. + +To do this, first [install GoReleaser](https://goreleaser.com/install/). + +Then, to create snapshot builds, do: + +```bash +goreleaser release --rm-dist --snapshot +``` + +If all goes according to plan, you should now have a bunch of multiple-architecture binaries and tars inside the `./dist` folder, and a snapshot Docker image should be built (check your terminal output for version). + +### Manually + +If you prefer a simple approach with fewer dependencies, you can also just build a Docker container manually in the following way: + +```bash +./scripts/build.sh && docker build -t superseriousbusiness/gotosocial:latest . +``` + ## Financial Compensation Right now there's no structure in place for financial compensation for pull requests and code. This is simply because there's no money being made on the project apart from the very small weekly Liberapay donations. diff --git a/Dockerfile b/Dockerfile index df2988368..5281ef667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,4 @@ -# STEP ONE: build the GoToSocial binary -FROM golang:1.17.1-alpine3.14 AS binary_builder -RUN apk update && apk upgrade --no-cache -RUN apk add git - -# create build dir -RUN mkdir -p /go/src/github.com/superseriousbusiness/gotosocial -WORKDIR /go/src/github.com/superseriousbusiness/gotosocial - -# move source files -ADD cmd /go/src/github.com/superseriousbusiness/gotosocial/cmd -ADD internal /go/src/github.com/superseriousbusiness/gotosocial/internal -ADD testrig /go/src/github.com/superseriousbusiness/gotosocial/testrig -ADD docs/swagger.go /go/src/github.com/superseriousbusiness/gotosocial/docs/swagger.go - -# dependencies and vendor -ADD go.mod /go/src/github.com/superseriousbusiness/gotosocial/go.mod -ADD go.sum /go/src/github.com/superseriousbusiness/gotosocial/go.sum -ADD vendor /go/src/github.com/superseriousbusiness/gotosocial/vendor - -# move .git dir and version for versioning -ADD .git /go/src/github.com/superseriousbusiness/gotosocial/.git -ADD version /go/src/github.com/superseriousbusiness/gotosocial/version - -# move the build script -ADD scripts/build.sh /go/src/github.com/superseriousbusiness/gotosocial/build.sh - -# do the build step -RUN ./build.sh - -# STEP TWO: build the web assets -FROM node:16.9.0-alpine3.14 AS web_builder -RUN apk update && apk upgrade --no-cache - -COPY web /web -WORKDIR /web/gotosocial-styling - -RUN yarn install -RUN BUILD_DIR=../assets node index.js - -# STEP THREE: bundle the admin webapp +# bundle the admin webapp FROM node:16.9.0-alpine3.14 AS admin_builder RUN apk update && apk upgrade --no-cache RUN apk add git @@ -49,19 +9,15 @@ WORKDIR /gotosocial-admin RUN npm install RUN node index.js -# STEP FOUR: build the final container FROM alpine:3.14.2 AS executor RUN apk update && apk upgrade --no-cache # copy over the binary from the first stage RUN mkdir -p /gotosocial/storage -COPY --from=binary_builder /go/src/github.com/superseriousbusiness/gotosocial/gotosocial /gotosocial/gotosocial +COPY gotosocial /gotosocial/gotosocial # copy over the web directory with templates etc -COPY --from=web_builder web /gotosocial/web - -# put the swagger yaml in the web assets directory so it can be accessed -COPY docs/api/swagger.yaml /gotosocial/web/assets/swagger.yaml +COPY web /gotosocial/web # copy over the admin directory COPY --from=admin_builder /gotosocial-admin/public /gotosocial/web/assets/admin diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index 40a2caa5e..17846686b 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -1679,7 +1679,7 @@ info: name: AGPL3 url: https://www.gnu.org/licenses/agpl-3.0.en.html title: GoToSocial - version: 0.1.0-SNAPSHOT + version: 0.0.1 paths: /api/v1/accounts: post: diff --git a/scripts/build.sh b/scripts/build.sh index 65b258cba..565435a69 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -2,8 +2,8 @@ set -eu -COMMIT=$(git rev-list -1 HEAD) -VERSION=$(cat ./version) +COMMIT="${COMMIT:-12345678}" +VERSION="${VERSION:-0.0.0}" CGO_ENABLED=0 go build -trimpath \ -tags 'netgo osusergo static_build' \ diff --git a/scripts/dockerbuild.sh b/scripts/dockerbuild.sh deleted file mode 100755 index a7e14d724..000000000 --- a/scripts/dockerbuild.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -eu - -BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" - -docker build -t "superseriousbusiness/gotosocial:${BRANCH_NAME}" . diff --git a/scripts/dockerpush.sh b/scripts/dockerpush.sh deleted file mode 100755 index fb952032c..000000000 --- a/scripts/dockerpush.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" - -docker push "superseriousbusiness/gotosocial:${BRANCH_NAME}" diff --git a/scripts/generateswagger.sh b/scripts/generateswagger.sh deleted file mode 100755 index a964ba41c..000000000 --- a/scripts/generateswagger.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -eu - -SWAGGER_FILE="docs/api/swagger.yaml" -GTS_VERSION="$(cat version)" - -swagger generate spec -o "${SWAGGER_FILE}" --scan-models -sed -i "s/REPLACE_ME/${GTS_VERSION}/" "${SWAGGER_FILE}" diff --git a/version b/version deleted file mode 100644 index 4ecb66440..000000000 --- a/version +++ /dev/null @@ -1 +0,0 @@ -0.1.0-SNAPSHOT \ No newline at end of file diff --git a/web/gotosocial-styling/index.js b/web/gotosocial-styling/index.js index 88d9398b2..43384a108 100644 --- a/web/gotosocial-styling/index.js +++ b/web/gotosocial-styling/index.js @@ -4,6 +4,7 @@ const Promise = require("bluebird"); const fs = require("fs").promises; const postcss = require('postcss'); const {parse} = require("postcss-scss"); +const argv = require('minimist')(process.argv.slice(2)); /* Bundle all postCSS files under the `templates/` directory separately, each prepended with the (variable) contents of ./colors.css @@ -42,10 +43,23 @@ function bundle([template, path]) { }); } -let buildDir = process.env.BUILD_DIR; +let buildDir + +// try reading from arguments first +if (argv["build-dir"] != undefined) { + buildDir = argv["build-dir"] +} + +// then try reading from environment variable +if (buildDir == undefined) { + buildDir = process.env.BUILD_DIR; +} + +// then take default if (buildDir == undefined) { buildDir = `${__dirname}/build`; } + console.log("bundling to", buildDir); function bundleAll() { diff --git a/web/gotosocial-styling/package.json b/web/gotosocial-styling/package.json index ae3d73e93..57594b091 100644 --- a/web/gotosocial-styling/package.json +++ b/web/gotosocial-styling/package.json @@ -7,6 +7,7 @@ "license": "AGPL-3.0", "dependencies": { "bluebird": "^3.7.2", + "minimist": "^1.2.5", "postcss": "^8.3.5", "postcss-color-function": "^4.1.0", "postcss-nested": "^4.2.1", diff --git a/web/gotosocial-styling/yarn.lock b/web/gotosocial-styling/yarn.lock index f61a02a53..1d39d5fd7 100644 --- a/web/gotosocial-styling/yarn.lock +++ b/web/gotosocial-styling/yarn.lock @@ -219,6 +219,11 @@ js-base64@^2.1.9: resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.6.4.tgz#f4e686c5de1ea1f867dbcad3d46d969428df98c4" integrity sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ== +minimist@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"