rewrite circleci config (#558)

* rewrite circleci config with better syntax

and try using grcov for coverage

* attempt to use less memory and to rerun if memory lack

* actually remove use of kcov

* use the right var

* try looping

* run test serialized

* use feature where we should and make macro path independant

* cargo fmt and remove some '"'

* add zip to dependancies

* reduce number of used thread after first failure

* push translations to crowdin on commit to master

* don't cache result of coverage
This commit is contained in:
fdb-hiroshima 2019-05-04 15:35:21 +02:00 committed by GitHub
parent 33a0c7dcd3
commit 918bda14ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 253 additions and 322 deletions

View File

@ -1,314 +1,253 @@
version: 2.1 version: 2.1
aliases: executors:
- &plume-docker default:
image: plumeorg/plume-buildenv:v0.0.5 parameters:
- &defaults postgres:
type: boolean
default: false
selenium:
type: boolean
default: false
docker: docker:
- *plume-docker - image: plumeorg/plume-buildenv:v0.0.7
- image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<</parameters.postgres>><<^parameters.postgres>>alpine:latest<</parameters.postgres>>
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
- image: <<#parameters.selenium>>elgalu/selenium:latest<</parameters.selenium>><<^parameters.selenium>>alpine:latest<</parameters.selenium>>
working_directory: ~/projects/Plume working_directory: ~/projects/Plume
- &postgresql
docker:
- *plume-docker
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
working_directory: ~/projects/Plume
- &selenium
docker:
- *plume-docker
- image: elgalu/selenium:latest
working_directory: ~/projects/Plume
- &postgresql_selenium
docker:
- *plume-docker
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
- image: elgalu/selenium:latest
working_directory: ~/projects/Plume
- &attach_workspace
attach_workspace:
at: ~/projects/Plume/
- &persist_to_workspace
persist_to_workspace:
root: ~/projects/Plume/
paths:
- ./
- &env_postgresql
environment: environment:
MIGRATION_DIRECTORY: migrations/postgres
FEATURES: postgres
DATABASE_URL: postgres://postgres@localhost/plume
RUST_TEST_THREADS: 1
- &env_sqlite
environment:
MIGRATION_DIRECTORY: migrations/sqlite
FEATURES: sqlite
DATABASE_URL: plume.sqlite3
RUST_TEST_THREADS: 1 RUST_TEST_THREADS: 1
FEATURES: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
DATABASE_URL: <<#parameters.postgres>>postgres://postgres@localhost/plume<</parameters.postgres>><<^parameters.postgres>>plume.sqlite<</parameters.postgres>>
- &restore_cache
restore_cache: commands:
keys: restore_env:
- v2-plume-notest-{{ checksum "Cargo.lock" }} description: checkout and pull cache
- v2-plume-notest parameters:
- &save_cache cache:
save_cache: type: enum
key: v2-plume-notest-{{ checksum "Cargo.lock" }} default: none
paths: enum: ["none", "clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
steps:
- checkout
- run: git config --global --remove-section url."ssh://git@github.com"
- restore_cache:
keys:
- v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
- v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-master
cache:
description: push cache
parameters:
cache:
type: enum
enum: ["clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
steps:
- save_cache:
key: v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
paths:
- ~/.cargo/ - ~/.cargo/
- ./target - ./target
- &restore_cache_web clippy:
restore_cache: description: run cargo clippy
keys: parameters:
- v2-plume-web-{{ checksum "Cargo.lock" }} package:
- v2-plume-web type: string
- &save_cache_web default: plume
save_cache: no_feature:
key: v2-plume-web-{{ checksum "Cargo.lock" }} type: boolean
paths: default: false
- ~/.cargo/
- ./target
- &restore_cache_cli
restore_cache:
keys:
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_cli
save_cache:
key: v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
paths:
- ~/.cargo/
- target
- &restore_cache_release
restore_cache:
keys:
- v2-plume-release-{{ checksum "Cargo.lock" }}
- v2-plume-release
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_release
save_cache:
key: v2-plume-release-{{ checksum "Cargo.lock" }}
paths:
- ~/.cargo
- target/release
- target/wasm32-unknown-unknown/release
- &restore_cache_plume_dead_code
restore_cache:
keys:
- v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
- v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}
- v2-plume-test-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_plume_dead_code
save_cache:
key: v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
paths:
- ~/.cargo/
- target
- &test_cli
steps: steps:
- *attach_workspace - run: cargo clippy <<^parameters.no_feature>>--no-default-features --features="${FEATURES}"<</parameters.no_feature>> --release -p <<parameters.package>> -- -D warnings
- run:
name: Set cache key
command: echo "$FEATURES" > /FEATURES
- *restore_cache_cli
- run:
name: clippy
command: cargo clippy --no-default-features --features="${FEATURES}" --release -p plume-cli -- -D warnings
- *save_cache_cli
- &test_unit
steps:
- *attach_workspace
- run:
name: Set cache key
command: echo "$FEATURES" > /FEATURES
- *restore_cache_plume_dead_code
- run:
name: clippy
command: cargo clippy --no-default-features --features="${FEATURES}" --release -- -D warnings
- run:
name: compile test
command: cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run || cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run
- run:
name: run test
command: ./script/run_unit_test.sh
- run:
name: upload coverage
command: ./script/upload_coverage.sh unit
- *save_cache_plume_dead_code
- &test_browser run_with_coverage:
description: run command with environment for coverage
parameters:
cmd:
type: string
steps: steps:
- *attach_workspace - run: |
- run: export RUSTFLAGS="-Zprofile -Zfewer-names -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads -Clink-arg=-Xlinker -Clink-arg=--no-keep-memory -Clink-arg=-Xlinker -Clink-arg=--reduce-memory-overheads"
name: Set cache key export CARGO_INCREMENTAL=0
command: echo "$FEATURES" > /FEATURES << parameters.cmd >>
- *restore_cache_plume_dead_code
- run:
name: install server
command: cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force || cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force
- run:
name: install plm
command: cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force || cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force
- run:
name: run test
command: ./script/run_browser_test.sh
environment:
BROWSER: firefox
- run:
name: upload coverage
command: ./script/upload_coverage.sh integration
- *save_cache_plume_dead_code
- &release upload_coverage:
description: merge coverage files and upload to codecov.io
parameters:
type:
type: string
steps: steps:
- *attach_workspace - run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'`
- *restore_cache_release - run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info
- run: - run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <<parameters.type>>
name: build frontend - run: find . -name 'plume*.gc*' -delete -o -name 'plm*.gc*' -delete
command: cargo web deploy -p plume-front --release - run: rm ccov.zip lcov.info
- run:
name: build server
command: cargo build --release --no-default-features --features="${FEATURES}" || cargo build --release --no-default-features --features="${FEATURES}"
- run:
name: build plm
command: cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli || cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli
- *save_cache_release
- run: script/generate_artifact.sh
- run: script/upload_test_environment.sh
- store_artifacts:
path: plume.tar.gz
destination: plume.tar.gz
build:
description: build a package
parameters:
package:
type: string
default: plume
release:
type: boolean
default: false
steps:
- run: |
cmd="cargo build <<#parameters.release>>--release<</parameters.release>> --no-default-features --features="${FEATURES}" -p <<parameters.package>> -j"
for i in 36 4 2 1 1; do
$cmd $i && exit 0
done
exit 1
jobs: jobs:
download_deps: cargo fmt:
<<: *defaults executor:
name: default
steps: steps:
- checkout - restore_env
- *attach_workspace - run: cargo fmt --all -- --check
- *restore_cache
- run: git config --global --remove-section url."ssh://git@github.com"
- run: cargo fetch
- *save_cache
- *persist_to_workspace
cargo_fmt: clippy:
<<: *defaults parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps: steps:
- *attach_workspace - restore_env:
- run: cargo fmt --all -- --check cache: clippy
- clippy
- clippy:
package: plume-cli
- clippy:
package: plume-front
no_feature: true
- cache:
cache: clippy
build_web: unit:
<<: *defaults parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps: steps:
- *attach_workspace - restore_env:
- *restore_cache_web cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: cargo clippy -p plume-front -- -D warnings - run_with_coverage:
- run: cargo web deploy -p plume-front cmd: |
- *save_cache_web cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j"
- *persist_to_workspace for i in 36 4 2 1 1; do
$cmd $i && break
done
cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -j1 -- --test-threads=1
- upload_coverage:
type: unit
- cache:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
test_cli_postgresql: integration:
<<: *postgresql parameters:
<<: *env_postgresql postgres:
<<: *test_cli type: boolean
executor:
test_cli_sqlite: name: default
<<: *defaults postgres: << parameters.postgres >>
<<: *env_sqlite selenium: true
<<: *test_cli
test_unit_postgresql:
<<: *postgresql
<<: *env_postgresql
<<: *test_unit
test_unit_sqlite:
<<: *defaults
<<: *env_sqlite
<<: *test_unit
test_browser_postgresql:
<<: *postgresql_selenium
<<: *env_postgresql
<<: *test_browser
test_browser_sqlite:
<<: *selenium
<<: *env_sqlite
<<: *test_browser
all_ok:
docker:
- image: alpine:3.7
steps: steps:
- run: /bin/true - restore_env:
release_postgresql: cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
<<: *defaults - run: cargo web deploy -p plume-front
<<: *env_postgresql - run_with_coverage:
<<: *release cmd: |
release_sqlite: cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . -j"
<<: *defaults for i in 36 4 2 1 1; do
<<: *env_sqlite $cmd $i && exit 0
<<: *release done
exit 1
- run_with_coverage:
cmd: |
cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli -j"
for i in 36 4 2 1 1; do
$cmd $i && exit 0
done
exit 1
- run:
name: run test
command: ./script/run_browser_test.sh
environment:
BROWSER: firefox
- upload_coverage:
type: integration
- cache:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
release:
parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps:
- restore_env:
cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: cargo web deploy -p plume-front --release
- build:
package: plume
release: true
- build:
package: plume-cli
release: true
- cache:
cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: ./script/generate_artifact.sh
- unless:
condition: << parameters.postgres >>
steps:
- run: ./script/upload_test_environment.sh
- store_artifacts:
path: plume.tar.gz
destination: plume.tar.gz
push translations:
executor:
name: default
steps:
- restore_env:
cache: none
- run: crowdin upload
workflows: workflows:
version: 2 version: 2
build_and_test: build and test:
jobs: jobs:
- download_deps - cargo fmt
- cargo_fmt: - clippy:
requires: postgres: false
- download_deps - clippy:
- build_web: postgres: true
requires: - unit:
- download_deps postgres: false
- test_cli_postgresql: - unit:
requires: postgres: true
- download_deps - integration:
- test_cli_sqlite: postgres: false
requires: - integration:
- download_deps postgres: true
- test_unit_postgresql: - release:
requires: postgres: false
- download_deps - release:
- test_unit_sqlite: postgres: true
requires: - push translations:
- download_deps filters:
- test_browser_postgresql: branches:
requires: only:
- build_web - /^master/
- test_browser_sqlite:
requires:
- build_web
- all_ok:
requires:
- cargo_fmt
- test_cli_postgresql
- test_cli_sqlite
- test_unit_postgresql
- test_unit_sqlite
- test_browser_postgresql
- test_browser_sqlite
- release_postgresql:
requires:
- all_ok
- release_sqlite:
requires:
- all_ok

View File

@ -3,8 +3,9 @@ ENV PATH="/root/.cargo/bin:${PATH}"
#install native/circleci/build dependancies #install native/circleci/build dependancies
RUN apt update &&\ RUN apt update &&\
apt install -y git ssh tar gzip ca-certificates &&\ apt install -y --no-install-recommends git ssh tar gzip ca-certificates default-jre&&\
apt install -y binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip unzip &&\
rm -rf /var/lib/apt/lists/*
#install and configure rust #install and configure rust
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\
@ -13,13 +14,10 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0
#compile some deps #compile some deps
RUN cargo install cargo-web &&\ RUN cargo install cargo-web &&\
cargo install grcov &&\
strip /root/.cargo/bin/* &&\
rm -fr ~/.cargo/registry rm -fr ~/.cargo/registry
#install coverage tools
RUN curl -L https://github.com/SimonKagstrom/kcov/archive/master.tar.gz | tar xz &&\
mkdir -p kcov-master/build && cd kcov-master/build && cmake .. && make &&\
make install && cd ../.. && rm -rf kcov-master
#set some compilation parametters #set some compilation parametters
COPY cargo_config /root/.cargo/config COPY cargo_config /root/.cargo/config
@ -29,3 +27,11 @@ RUN pip3 install selenium
#install and configure caddy #install and configure caddy
RUN curl https://getcaddy.com | bash -s personal RUN curl https://getcaddy.com | bash -s personal
COPY Caddyfile /Caddyfile COPY Caddyfile /Caddyfile
#install crowdin
RUN mkdir /crowdin && cd /crowdin &&\
curl -O https://downloads.crowdin.com/cli/v2/crowdin-cli.zip &&\
unzip crowdin-cli.zip && rm crowdin-cli.zip &&\
cd * && mv crowdin-cli.jar /usr/local/bin && cd && rm -rf /crowdin &&\
/bin/echo -e '#!/bin/sh\njava -jar /usr/local/bin/crowdin-cli.jar $@' > /usr/local/bin/crowdin &&\
chmod +x /usr/local/bin/crowdin

View File

@ -1,3 +1,3 @@
[target.x86_64-unknown-linux-gnu] [target.x86_64-unknown-linux-gnu]
# link dead code for coverage, attempt to reduce linking memory usage to not get killed # link dead code for coverage, attempt to reduce linking memory usage to not get killed
rustflags = ["-Clink-dead-code", "-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"] rustflags = ["-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"]

View File

@ -1,3 +1,5 @@
"project_identifier": "plume"
"api_key_env": CROWDIN_API_KEY
files: files:
- source: /po/plume/plume.pot - source: /po/plume/plume.pot
translation: /po/plume/%two_letters_code%.po translation: /po/plume/%two_letters_code%.po

View File

@ -8,6 +8,7 @@ use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2; use proc_macro2::TokenStream as TokenStream2;
use std::fs::{read_dir, File}; use std::fs::{read_dir, File};
use std::io::Read; use std::io::Read;
use std::path::Path;
use std::str::FromStr; use std::str::FromStr;
#[proc_macro] #[proc_macro]
@ -20,7 +21,12 @@ pub fn import_migrations(input: TokenStream) -> TokenStream {
} else { } else {
"migrations" "migrations"
}; };
let mut files = read_dir(migration_dir) let path = Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
.find(|path| path.join(migration_dir).is_dir() || path.join(".git").exists())
.expect("migrations dir not found")
.join(migration_dir);
let mut files = read_dir(path)
.unwrap() .unwrap()
.map(|dir| dir.unwrap()) .map(|dir| dir.unwrap())
.filter(|dir| dir.file_type().unwrap().is_dir()) .filter(|dir| dir.file_type().unwrap().is_dir())

View File

@ -3,16 +3,12 @@ set -eo pipefail
export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
mkdir -p "target/cov/plume" plm migration run
mkdir -p "target/cov/plm" plm migration redo
plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm' plm instance new -d plume-test.local -n plume-test
plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
$plm migration run plume &
$plm migration redo
$plm instance new -d plume-test.local -n plume-test
$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume &
caddy -conf /Caddyfile & caddy -conf /Caddyfile &
until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null
@ -22,4 +18,4 @@ python3 -m unittest *.py
kill -SIGINT %1 kill -SIGINT %1
kill -SIGKILL %2 kill -SIGKILL %2
wait sleep 15

View File

@ -1,15 +0,0 @@
#!/bin/bash
set -eo pipefail
for file in target/debug/*-*[^\.d]; do
if [[ -x "$file" ]]
then
filename=$(basename $file)
if [[ $filename =~ ^plume_macro ]]; then
rm $file
continue
fi
mkdir -p "target/cov/$filename"
kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file"
rm $file
fi
done

View File

@ -1,3 +0,0 @@
#!/bin/bash
set -eo pipefail
bash <(curl -s https://codecov.io/bash) -F $1