From 4cfef2e130c1ec31f57dc685624ccc57336f14e5 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:17:15 +0200 Subject: [PATCH 1/8] Vagrant: introducing epub VM Adding separate VM for publication tasks (e.g.: "make site"). --- Vagrantfile | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 6a67c53b..c913025d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -41,6 +41,66 @@ $frontend = <<-SHELL SHELL +$epub = <<-SHELL + #!/bin/sh -eux + + # Enable the NodeSource repository + curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash - + + # Add yarn to apt-get + curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | tee /usr/share/keyrings/yarnkey.gpg >/dev/null + echo "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" | tee /etc/apt/sources.list.d/yarn.list + + apt-get update && sudo apt-get install yarn + + # Install system deps + DEBIAN_FRONTEND=noninteractive apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + nodejs \ + graphviz \ + make \ + yarn \ + texlive-latex-base \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + latexmk \ + texlive-xetex \ + fonts-lmodern \ + fonts-open-sans \ + fonts-dejavu \ + build-essential \ + ca-certificates \ + git \ + libdbus-1-3 \ + libfontconfig \ + libx11-xcb-dev \ + wget \ + libc6-dev + + # Install GNAT Community + git clone https://github.com/AdaCore/gnat_community_install_script.git /gnat_installer/script \ + && wget -q https://community.download.adacore.com/v1/f3a99d283f7b3d07293b2e1d07de00e31e332325?filename=gnat-2021-20210519-x86_64-linux-bin -O /gnat_installer/actual \ + && sh /gnat_installer/script/install_package.sh /gnat_installer/actual /gnat com.adacore.spark2014_discovery,com.adacore.gnat \ + && rm -rf /gnat_installer + + echo 'export PATH="/gnat/bin:${PATH}"' >> /home/vagrant/.bashrc + source /home/vagrant/.bashrc + + # Install learn deps + python3 -m venv /vagrant/venv + source /vagrant/venv/bin/activate + pip3 install -r /vagrant/frontend/requirements.txt + + cd /vagrant/frontend + yarn + +SHELL + $backend = <<-SHELL #!/bin/sh -eux @@ -87,7 +147,6 @@ $backend = <<-SHELL SHELL Vagrant.configure("2") do |config| - config.vm.box = "bento/ubuntu-18.04" config.vm.provider "virtualbox" do |vb| vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"] @@ -96,6 +155,7 @@ Vagrant.configure("2") do |config| config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.define "web" do |web| + web.vm.box = "bento/ubuntu-18.04" web.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1" web.vm.synced_folder './frontend', '/vagrant/frontend' @@ -104,7 +164,17 @@ Vagrant.configure("2") do |config| web.vm.provision :shell, inline: $frontend end + config.vm.define "epub" do |epub| + epub.vm.box = "bento/ubuntu-21.04" + + epub.vm.synced_folder './frontend', '/vagrant/frontend' + epub.vm.synced_folder './content', '/vagrant/content' + + epub.vm.provision :shell, inline: $epub + end + config.vm.define "server" do |server| + server.vm.box = "bento/ubuntu-18.04" server.vm.network "forwarded_port", guest: 8000, host: 8000, host_ip: "127.0.0.1" server.vm.network "forwarded_port", guest: 6379, host: 6379, host_ip: "127.0.0.1" From 4a4cf902e5aedc18b4c3a6bdc623df190b72199f Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:17:46 +0200 Subject: [PATCH 2/8] README: adding short description of epub VM --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 35e4bb96..8f440c54 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,16 @@ To setup for development run: ``` $ vagrant up ``` -This will spin up two vms: +This will spin up three vms: web: Is the the build system for the frontend web content. This includes the webpack build system and sphinx build. server: Is the backend server with the widget API. +epub: Is the publishing server. This includes all packages needed to +generate the learn website. + To build and start the development server for the frontend, run: ``` $ vagrant ssh web @@ -54,3 +57,16 @@ $ ./dev_server.sh You can use ctrl-c to quit the bash script which will kill both the flask and celery processes. + +To build and start the publishing server, run: +``` +$ vagrant ssh epub + +# The following commands will be run inside the vm + +$ cd /vagrant +$ source venv/bin/activate +$ make site +``` +This will build the content for the learn website. You can find it in the +`/vagrant/frontend/dist` directory. From 8774d61acbffdb4247ae823f7bf570079d79a859 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:18:45 +0200 Subject: [PATCH 3/8] Makefile: adding test_content for site and site-staging targets --- frontend/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/Makefile b/frontend/Makefile index 33484329..a3827356 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -140,11 +140,11 @@ webpack-staging: yarn run staging # Build the site pointing to 'cloudchecker.learn.r53.adacore.com' -site: cleanall webpack-production pdf_books +site: cleanall webpack-production test_content pdf_books CODE_SERVER_URL="https://cloudchecker.learn.r53.adacore.com" GEN_LEARN_SITE=yes $(SPHINXBUILD) -M html $(CONTENT_DIR) \ "$(BUILDDIR)" $(SPHINXOPTS) $(O) -v -c "$(SPHINXCONF)" -site-staging: cleanall webpack-staging pdf_books +site-staging: cleanall webpack-staging test_content pdf_books $(SPHINXBUILD) -M html $(CONTENT_DIR) \ "$(BUILDDIR)" $(SPHINXOPTS) $(O) -v -c "$(SPHINXCONF)" From 4a075dd46607c8dec192aee0b071148ad3700d43 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:19:52 +0200 Subject: [PATCH 4/8] GitHub actions: upgrading to Ubuntu 21.04 for Sphinx content test --- .github/workflows/sphinx-content-tests.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sphinx-content-tests.js.yml b/.github/workflows/sphinx-content-tests.js.yml index 203f9b7f..1cac6b73 100644 --- a/.github/workflows/sphinx-content-tests.js.yml +++ b/.github/workflows/sphinx-content-tests.js.yml @@ -9,7 +9,7 @@ defaults: jobs: sphinx-content: - runs-on: ubuntu-18.04 + runs-on: ubuntu-21.04 strategy: matrix: From 21c24a914dd8a846c44b5f6161bc70ce0de4157c Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:21:17 +0200 Subject: [PATCH 5/8] GitHub actions: grouping steps Using "make site" for everything. --- .github/workflows/sphinx-content-tests.js.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sphinx-content-tests.js.yml b/.github/workflows/sphinx-content-tests.js.yml index 1cac6b73..751ebc50 100644 --- a/.github/workflows/sphinx-content-tests.js.yml +++ b/.github/workflows/sphinx-content-tests.js.yml @@ -55,10 +55,8 @@ jobs: pip install -r requirements.txt - name: Run SPHINX engine tests run: make SPHINXOPTS="-W" test_engine - - name: Run SPHINX content tests - run: make -k test_content - - name: Build PDF books including build/runtime output - run: make pdf_books + - name: Build learn site + run: make site - name: Archive PDF books in artifact uses: actions/upload-artifact@v2 with: From 024d6f7a5515eb5c23fc0d2b65d473f6d5d59290 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:22:00 +0200 Subject: [PATCH 6/8] GitHub actions: adding artifact for HTML content --- .github/workflows/sphinx-content-tests.js.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/sphinx-content-tests.js.yml b/.github/workflows/sphinx-content-tests.js.yml index 751ebc50..3a488465 100644 --- a/.github/workflows/sphinx-content-tests.js.yml +++ b/.github/workflows/sphinx-content-tests.js.yml @@ -64,3 +64,10 @@ jobs: path: | frontend/dist/pdf_books retention-days: 1 + - name: Archive HTML files in artifact + uses: actions/upload-artifact@v2 + with: + name: learn-html + path: | + frontend/dist/html + retention-days: 1 From 14d6afa658ce1d59d43c25be5cf5de914d359a6f Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:27:05 +0200 Subject: [PATCH 7/8] GitHub actions: replacing OS Just for testing / Ubuntu 21.04 doesn't seem to be available. --- .github/workflows/sphinx-content-tests.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sphinx-content-tests.js.yml b/.github/workflows/sphinx-content-tests.js.yml index 3a488465..9853aec1 100644 --- a/.github/workflows/sphinx-content-tests.js.yml +++ b/.github/workflows/sphinx-content-tests.js.yml @@ -9,7 +9,7 @@ defaults: jobs: sphinx-content: - runs-on: ubuntu-21.04 + runs-on: ubuntu-20.04 strategy: matrix: From 73cfc0cd1161b6c4aa2855474980d2c6659b4d41 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Sat, 14 Aug 2021 01:58:12 +0200 Subject: [PATCH 8/8] GitHub Actions: using multiple steps again Reverting previous change and adding step for Webpack production. --- .github/workflows/sphinx-content-tests.js.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sphinx-content-tests.js.yml b/.github/workflows/sphinx-content-tests.js.yml index 9853aec1..ec780428 100644 --- a/.github/workflows/sphinx-content-tests.js.yml +++ b/.github/workflows/sphinx-content-tests.js.yml @@ -55,8 +55,12 @@ jobs: pip install -r requirements.txt - name: Run SPHINX engine tests run: make SPHINXOPTS="-W" test_engine - - name: Build learn site - run: make site + - name: Run Webpack production + run: make cleanall webpack-production + - name: Run SPHINX content tests + run: make -k test_content + - name: Build PDF books including build/runtime output + run: make pdf_books - name: Archive PDF books in artifact uses: actions/upload-artifact@v2 with: