Merge pull request #601 from gusthoff/topic/infrastructure/sphinx/pdf_books/20210814

Frontend: improvements for site publication

.github/workflows/sphinx-content-tests.js.yml

@@ -9,7 +9,7 @@ defaults:
 jobs:
   sphinx-content:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     strategy:
       matrix:
@@ -55,6 +55,8 @@ jobs:
         pip install -r requirements.txt
     - name: Run SPHINX engine tests
       run: make SPHINXOPTS="-W" test_engine
+    - name: Run Webpack production
+      run: make cleanall webpack-production
     - name: Run SPHINX content tests
       run: make -k test_content
     - name: Build PDF books including build/runtime output
@@ -66,3 +68,10 @@ jobs:
         path: |
           frontend/dist/pdf_books
         retention-days: 1
+    - name: Archive HTML files in artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: learn-html
+        path: |
+          frontend/dist/html
+        retention-days: 1

README.md

@@ -18,13 +18,16 @@ To setup for development run:
 ```
 $ vagrant up
 ```
-This will spin up two vms:
+This will spin up three vms:
 
 web: Is the the build system for the frontend web content. This includes the
 webpack build system and sphinx build.
 
 server: Is the backend server with the widget API.
 
+epub: Is the publishing server. This includes all packages needed to
+generate the learn website.
+
 To build and start the development server for the frontend, run:
 ```
 $ vagrant ssh web
@@ -54,3 +57,16 @@ $ ./dev_server.sh
 
 You can use ctrl-c to quit the bash script which will kill both the flask
 and celery processes.
+
+To build and start the publishing server, run:
+```
+$ vagrant ssh epub
+
+# The following commands will be run inside the vm
+
+$ cd /vagrant
+$ source venv/bin/activate
+$ make site
+```
+This will build the content for the learn website. You can find it in the
+`/vagrant/frontend/dist` directory.

Vagrantfile

@@ -41,6 +41,66 @@ $frontend = <<-SHELL
 
 SHELL
 
+$epub = <<-SHELL
+  #!/bin/sh -eux
+
+  # Enable the NodeSource repository
+  curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+
+  # Add yarn to apt-get
+  curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | tee /usr/share/keyrings/yarnkey.gpg >/dev/null
+  echo "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+  apt-get update && sudo apt-get install yarn
+
+  # Install system deps
+  DEBIAN_FRONTEND=noninteractive apt-get update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      python3 \
+      python3-pip \
+      python3-venv \
+      nodejs \
+      graphviz \
+      make \
+      yarn \
+      texlive-latex-base \
+      texlive-latex-recommended \
+      texlive-latex-extra \
+      texlive-fonts-recommended \
+      texlive-fonts-extra \
+      latexmk \
+      texlive-xetex \
+      fonts-lmodern \
+      fonts-open-sans \
+      fonts-dejavu \
+      build-essential \
+      ca-certificates \
+      git \
+      libdbus-1-3 \
+      libfontconfig \
+      libx11-xcb-dev \
+      wget \
+      libc6-dev
+
+  # Install GNAT Community
+  git clone https://github.com/AdaCore/gnat_community_install_script.git /gnat_installer/script \
+    && wget -q https://community.download.adacore.com/v1/f3a99d283f7b3d07293b2e1d07de00e31e332325?filename=gnat-2021-20210519-x86_64-linux-bin -O /gnat_installer/actual \
+    && sh /gnat_installer/script/install_package.sh /gnat_installer/actual /gnat com.adacore.spark2014_discovery,com.adacore.gnat \
+    && rm -rf /gnat_installer
+
+  echo 'export PATH="/gnat/bin:${PATH}"' >> /home/vagrant/.bashrc
+  source /home/vagrant/.bashrc
+
+  # Install learn deps
+  python3 -m venv /vagrant/venv
+  source /vagrant/venv/bin/activate
+  pip3 install -r /vagrant/frontend/requirements.txt
+
+  cd /vagrant/frontend
+  yarn
+
+SHELL
+
 $backend = <<-SHELL
   #!/bin/sh -eux
 
@@ -87,7 +147,6 @@ $backend = <<-SHELL
 SHELL
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-18.04"
 
   config.vm.provider "virtualbox" do |vb|
     vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"]
@@ -96,6 +155,7 @@ Vagrant.configure("2") do |config|
   config.vm.synced_folder '.', '/vagrant', disabled: true
 
   config.vm.define "web" do |web|
+    web.vm.box = "bento/ubuntu-18.04"
     web.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
 
     web.vm.synced_folder './frontend', '/vagrant/frontend'
@@ -104,7 +164,17 @@ Vagrant.configure("2") do |config|
     web.vm.provision :shell, inline: $frontend
   end
 
+  config.vm.define "epub" do |epub|
+    epub.vm.box = "bento/ubuntu-21.04"
+
+    epub.vm.synced_folder './frontend', '/vagrant/frontend'
+    epub.vm.synced_folder './content', '/vagrant/content'
+
+    epub.vm.provision :shell, inline: $epub
+  end
+
   config.vm.define "server" do |server|
+    server.vm.box = "bento/ubuntu-18.04"
 
     server.vm.network "forwarded_port", guest: 8000, host: 8000, host_ip: "127.0.0.1"
     server.vm.network "forwarded_port", guest: 6379, host: 6379, host_ip: "127.0.0.1"

frontend/Makefile

@@ -140,11 +140,11 @@ webpack-staging:
 	yarn run staging
 
 # Build the site pointing to 'cloudchecker.learn.r53.adacore.com'
-site: cleanall webpack-production pdf_books
+site: cleanall webpack-production test_content pdf_books
 	CODE_SERVER_URL="https://cloudchecker.learn.r53.adacore.com" GEN_LEARN_SITE=yes $(SPHINXBUILD) -M html $(CONTENT_DIR) \
         "$(BUILDDIR)" $(SPHINXOPTS) $(O) -v -c "$(SPHINXCONF)"
 
-site-staging: cleanall webpack-staging pdf_books
+site-staging: cleanall webpack-staging test_content pdf_books
 	$(SPHINXBUILD) -M html $(CONTENT_DIR) \
         "$(BUILDDIR)" $(SPHINXOPTS) $(O) -v -c "$(SPHINXCONF)"