From 39b0607f51e4c32d9626d96da634a8c2e8239dd9 Mon Sep 17 00:00:00 2001 From: Bill Stumbo Date: Tue, 11 Jun 2024 23:41:06 -0400 Subject: [PATCH 1/4] Additional cleanup --- static/robots.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/robots.txt b/static/robots.txt index 14267e903..ba0d14348 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,2 +1,5 @@ +# Placeholder file. Overwritten in production by setting enableRobotsTXT to true +# See config/production/hugo.yaml +# User-agent: * -Allow: / \ No newline at end of file +Disallow: / \ No newline at end of file From e08bef4eb49035f5d7490065e88ab9590641a2c0 Mon Sep 17 00:00:00 2001 From: Bill Stumbo Date: Thu, 5 Sep 2024 06:19:44 -0400 Subject: [PATCH 2/4] Fix Robots.txt to be automatically generated for Production --- .github/workflows/gh-pages.yml | 21 +++++++++++++-------- layouts/robots.txt | 4 ++++ static/robots.txt | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 layouts/robots.txt diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 5b5d2cb3c..9272d79df 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -74,6 +74,10 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' || needs.check.outputs.cacheHit != 'true' steps: + - uses: actions/setup-go@v5 + with: + go-version: '1.18.1' + - uses: actions/checkout@v4 - name: Cache Zotero Bibliography @@ -107,7 +111,7 @@ jobs: - name: Setup Hugo uses: peaceiris/actions-hugo@v3 with: - hugo-version: '0.127.0' + hugo-version: '0.133.1' extended: true - name: Setup Node @@ -120,11 +124,12 @@ jobs: - run: npm install --verbose - name: Build - run: hugo -e $hugoEnvironment + run: hugo -e $hugoEnvironment -d ~/temp/deploy - - name: Deploy - uses: peaceiris/actions-gh-pages@v4 - if: github.ref == 'refs/heads/main' - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./public + +# - name: Deploy +# uses: peaceiris/actions-gh-pages@v4 +# if: github.ref == 'refs/heads/main' +# with: +# github_token: ${{ secrets.GITHUB_TOKEN }} +# publish_dir: ./public diff --git a/layouts/robots.txt b/layouts/robots.txt new file mode 100644 index 000000000..f59e94455 --- /dev/null +++ b/layouts/robots.txt @@ -0,0 +1,4 @@ +# Production robots.txt +# +User-agent: * +Allow: / diff --git a/static/robots.txt b/static/robots.txt index ba0d14348..2fe14329d 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -2,4 +2,4 @@ # See config/production/hugo.yaml # User-agent: * -Disallow: / \ No newline at end of file +Disallow: / From dd49c86f6b1da7e00942e2d59481eb4f369ea334 Mon Sep 17 00:00:00 2001 From: Bill Stumbo Date: Tue, 17 Sep 2024 23:32:33 -0400 Subject: [PATCH 3/4] Fix robots.txt (#238) Fix automatic generation of robots.txt. Validate on staging site and fix staging yaml code to work correctly with GitHub pages Update GitHub Action replacing PeaceIris actions with vanilla GitHub actions; align with Hugo's documented deployment process. --------- Signed-off-by: Bill Stumbo --- .github/workflows/gh-pages.yml | 71 ++++++++++++++++++++++++---------- .gitignore | 1 - config/_default/hugo.yaml | 2 +- config/_default/markup.yaml | 2 +- config/_default/params.yaml | 2 +- config/_default/privacy.yaml | 2 +- config/staging/hugo.yaml | 8 +++- 7 files changed, 61 insertions(+), 27 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 9272d79df..2904fa225 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,13 +25,26 @@ on: push: branches: - main # Set a branch to deploy + schedule: - cron: "0 3 * * *" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +defaults: + run: + shell: bash + env: # ---------------------------------------------------------------------------- # Specify the deployment environment: staging or production - hugoEnvironment: production + HUGO_ENVIRONMENT: staging + HUGO_VERSION: 0.133.1 jobs: # ---------------------------------------------------------------------------- @@ -42,6 +55,7 @@ jobs: outputs: zoteroVersion: ${{ fromJson(steps.zoteroVersion.outputs.headers).last-modified-version }} cacheHit: ${{ steps.cache-zotero-bib.outputs.cache-hit }} + runs-on: ubuntu-latest concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -66,19 +80,18 @@ jobs: ${{ fromJson(steps.zoteroVersion.outputs.headers).last-modified-version }} # ---------------------------------------------------------------------------- - # Deploy the website. This job is conditional, we will always run it on a + # Build the website. This job is conditional, we will always run it on a # push or if on a scheduled run the cache was determined to be out of date. # - deploy: + build: needs: check runs-on: ubuntu-latest if: github.event_name == 'push' || needs.check.outputs.cacheHit != 'true' steps: - - uses: actions/setup-go@v5 - with: - go-version: '1.18.1' - - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 - name: Cache Zotero Bibliography id: cache-zotero-bib @@ -106,13 +119,17 @@ jobs: ./update_bibliography.sh sudo cp --recursive ${GITHUB_WORKSPACE}/static/data ~/data fi - shell: bash - - name: Setup Hugo - uses: peaceiris/actions-hugo@v3 - with: - hugo-version: '0.133.1' - extended: true + # Install Hugo Extended + # + - name: Install Hugo CLI + run: | + wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ + && sudo dpkg -i ${{ runner.temp }}/hugo.deb + + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 - name: Setup Node uses: actions/setup-node@v4 @@ -124,12 +141,26 @@ jobs: - run: npm install --verbose - name: Build - run: hugo -e $hugoEnvironment -d ~/temp/deploy + env: +# HUGO_ENVIRONMENT: staging + HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache + TZ: America/New York + run: hugo -d "./public" --cleanDestinationDir -e staging + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./public + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build -# - name: Deploy -# uses: peaceiris/actions-gh-pages@v4 -# if: github.ref == 'refs/heads/main' -# with: -# github_token: ${{ secrets.GITHUB_TOKEN }} -# publish_dir: ./public + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + diff --git a/.gitignore b/.gitignore index b41f2aa34..3ad067183 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ _site .sass-cache .jekyll-metadata -public/ node_modules/ _gen/ .hugo_build.lock diff --git a/config/_default/hugo.yaml b/config/_default/hugo.yaml index 7e625d5e8..f3a589831 100644 --- a/config/_default/hugo.yaml +++ b/config/_default/hugo.yaml @@ -17,7 +17,7 @@ canonifyURLs: false # Default to false and set to true for production in config/production/hugo.yaml enableRobotsTXT: false -# aasetDir: Location where Hugo looks for assets +# assetDir: Location where Hugo looks for assets assetDir: static # Enable .GitInfo object for each page. This will give values to .Lastmod etc. diff --git a/config/_default/markup.yaml b/config/_default/markup.yaml index 4b348b3fc..01caed2c5 100644 --- a/config/_default/markup.yaml +++ b/config/_default/markup.yaml @@ -14,7 +14,7 @@ goldmark: # See: https://gohugo.io/getting-started/configuration-markup/#highlight # for additional information on configuration options. # https://www.docsy.dev/docs/adding-content/lookandfeel/#code-highlighting-with-chroma -# contains Docsy specfiic code highlighting information +# contains Docsy specfic code highlighting information highlight: # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html #style: solarized-dark diff --git a/config/_default/params.yaml b/config/_default/params.yaml index 759ed2d31..1e6030e2c 100644 --- a/config/_default/params.yaml +++ b/config/_default/params.yaml @@ -53,7 +53,7 @@ github_branch: main github_subdir: github_project_repo: https://github.com/interlisp/medley -# Google custom seach engine configuration +# Google custom search engine configuration # gcs_engine_id: search engine gcs_engine_id: 33ef4cbe0703b4f3a diff --git a/config/_default/privacy.yaml b/config/_default/privacy.yaml index 527ab0b56..0c4f9e895 100644 --- a/config/_default/privacy.yaml +++ b/config/_default/privacy.yaml @@ -5,7 +5,7 @@ # See: https://gohugo.io/about/hugo-and-gdpr/ # # googleAnalytics: -# ananymizIP: Enable anonymiation of IP addresses +# ananymizeIP: Enable anonymization of IP addresses # disable: Set to true to disable googleAnalytics # respectDoNotTrack: Check for Do Not Track in headers # useSessionStorage: Store session information in storage and diff --git a/config/staging/hugo.yaml b/config/staging/hugo.yaml index dcf0c6a77..904e9b617 100644 --- a/config/staging/hugo.yaml +++ b/config/staging/hugo.yaml @@ -1,4 +1,8 @@ -baseURL: https://wasm.interlisp.org +baseURL: https://stumbo.github.io/InterlispDraft.github.io/ + languageCode: en-us + +# title +# Insert Staging Environment onto every page to make clear +# this is not the production site title: 'Staging Environment' -publishDir: stage From 4c5320aa2cab2bcb0894d04be54046bbaf546a72 Mon Sep 17 00:00:00 2001 From: Bill Stumbo Date: Thu, 19 Sep 2024 00:03:20 -0400 Subject: [PATCH 4/4] Set to Production. Update Readme Add section on Staging --- .github/workflows/gh-pages.yml | 7 +- .gitignore | 1 + README.md | 116 ++++++++++++++++++++++++++++----- config/_default/privacy.yaml | 2 +- 4 files changed, 104 insertions(+), 22 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2904fa225..0af0cb522 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -43,7 +43,7 @@ defaults: env: # ---------------------------------------------------------------------------- # Specify the deployment environment: staging or production - HUGO_ENVIRONMENT: staging + HUGO_ENVIRONMENT: production HUGO_VERSION: 0.133.1 jobs: @@ -142,10 +142,9 @@ jobs: - name: Build env: -# HUGO_ENVIRONMENT: staging HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache TZ: America/New York - run: hugo -d "./public" --cleanDestinationDir -e staging + run: hugo --cleanDestinationDir -e $HUGO_ENVIRONMENT - name: Upload artifact uses: actions/upload-pages-artifact@v3 @@ -163,4 +162,4 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 - + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3ad067183..b41f2aa34 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ _site .sass-cache .jekyll-metadata +public/ node_modules/ _gen/ .hugo_build.lock diff --git a/README.md b/README.md index 8883a1a93..fb23a47ac 100644 --- a/README.md +++ b/README.md @@ -86,35 +86,117 @@ reflection of the bibliographic material related to Medley and Interlisp. Building the website is driven by a GitHub workflow. -The workflow is trigger by one of two events, a `push` to main, representing updates +The workflow is triggered by one of two events, a `push` to main, representing updates to the Interlisp.org website or a scheduled execution of the workflow. The workflow is scheduled to run on a regular basis to ensure the bibliography remains consistent with the online Zotero catalog. -The workflow consists of two jobs. The first job, `check`, uses Zotero's REST -interface to query for the latest version of the group bibliography. The call -made is a `GET` call to `https://api.zotero.org/groups/2914042/items`. This call +The GitHub Action workflow can also be initiated from the Action panel within +the Interlisp.github.io repository. This option allows manual execution when +necessary. + +The workflow consists of three jobs. The first job, `check`, uses Zotero's REST +interface to query for the latest version of the group bibliography. A `GET` call +is made to `https://api.zotero.org/groups/2914042/items`. It returns a collection of metadata and information describing the current state of items within the catalog. We are interested in a specific header, `Last-Modified-Version`. The value returned with this header is incremented every time the Zotero Interlisp -catalog is updated. The value returned is used as a cache-key for a cached -version of the json file of the bibliography we create. The first job completes -by providing the current Zotero version and whether the cache needs to be updated. +catalog is updated. We use the value returned as a cache-key for the bibliography. +If the cache-key matches one in the current GitHub Action cache we use the saved +bibliography information and save the overhead of building it. -The second job, `deploy`, starts by determining if a deploy needs to occur. If -the workflow was initiated by a `push` a deploy will always be done. However, -if the workflow was started by a scheduled execution, if the Zotero bibliography -cache is consistent with the online Zotero catalog, the deploy is skipped. +The second job, `build`, starts by determining if a build and deploy need to occur. +If the workflow was initiated by a `push` a deploy will always be done. However, +if the workflow was started by a scheduled execution and the Zotero bibliography +cache is consistent with the online Zotero catalog, the build and deploy are skipped. -A deploy starts by checking out the `Interlisp.github.io` repository. Then, -if the cache is valid, the contents are copied into the `data` file within the +A build starts by checking out the `Interlisp.github.io` repository. Then, +if the Zotereo cache is valid, its contents are copied into the `data` file within the repository directory structure. If the cache is invalid, the `update_bibliography.sh` -shell script is run to download a new copy of the bibliography as a `json` file. +shell script runs and downloads a new copy of the bibliography as a `json` file. Once downloaded the script does some additional processing to complete the formatting of the file. -Once this work is completed, Hugo is setup and the website is deployed. +After this work is completed, Hugo is setup and run to build the website. We use +Hugo extended to build our site. The version of Hugo currently being used is +defined by the environment variable, `HUGO_VERSION`. + +We run Hugo with two flags: + +- `-e $HUGO_ENVIRONMENT` to specify whether we are building a production or staging site. If the website is being build to deploy to Interlisp.org,it should be built with `HUGO_ENVIRONMENT` set to production. Deployment to any other site should set the environment flag to staging. +- `--cleanDestinationDir` clears the destination directory, `./public` on each build. This will ensure we do not have any unneeded artifacts in our deployment. + +The last part of the build activity is to save the created artifact, the information +in the `./public` directory. We use the GitHub composition action `upload-pages-artifact` +for this. It packages the contents of the directory and stores it in the appropriate +format for deployment to GitHub pages. + +The last job in the tool chain is `deploy`. This job simply takes the output +of the build step and formally deploys it to GitHub pages using the GitHub `deploy-pages` +action. + +### Deploying a Staging Site + +Successfully deploying a Staging Site requires you to configure your GitHub +repository to enable GitHub Pages. The following steps will accomplish this task: + +1. Clone the Interlisp.github.io repository into your GitHub site +2. In GitHub go to the cloned repository, in my case https://github.com/stumbo/InterlispDraft.github.io and select Settings +3. Under Settings, find Pages and select it +4. Under **Build and deployment** set Source to Deploy *GitHub Actions* + +Once the repository is cloned and GitHub Pages has been setup, you can deploy a +staging site to validate changes prior to creating a Pull Request to merge your +changes back into the main site. + +When creating a staging site we want to do a couple things to ensure we do not +interfere with the production site, first we want to disable Google Analytics +and secondly we want to ensure the site is not crawled and indexed. + +#### Setup Your Repository + +A best practice for the updating your clone of the repository is to create a branch +and make the following required changes on the branch you created. + +The appropriate settings for this are all enabled by setting the `HUGO_ENVIRONMENT` +variable in `.github/workflows/gh-pages.yaml` to *staging*. + +You also need to set `baseURL` to match the GitHub site you are deploying to +in the `config/staging/hugo.yaml` file. The file currently looks like: + +```yaml +baseURL: https://stumbo.github.io/InterlispDraft.github.io/ + +languageCode: en-us + +# title +# Insert Staging Environment onto every page to make clear +# this is not the production site +title: 'Staging Environment' +``` + +Make sure the `baseURL` reflects the complete path of your repository. Failure +to do this will either cause the deployment to fail or URLs within your built +site may be incorrectly set. Resulting in 404s or expected resources not found. + +With these changes the cloned repository is ready to be deployed to a staging site. + +Commit the changes you made and push the new branch to your cloned repository. +At this point, create a Pull Request to merge the changes you made into your +repository's main branch. Complete the operation by merging the pull request. + +Once the merge occurs, the GitHub Actions should fire off and your site will be +built and deployed. + +Once you have successfully completed this operation and your staging site is +deployed and operational you can experiment with adding new content or +functionality to the Interlisp site. + +#### Develop a Feature +To develop new pages or functionality, create a new branch for your work. once +you have completed development and testing on your staging site, you can create +a PR to merge the content into the Interlisp site. ### Running Hugo and Docsy Locally @@ -233,7 +315,7 @@ that have components specific to `Interlisp.github.io` are as follows: - `documentation` - contains the pdf files referenced in the document section of the home page - `favicons` - contains `favicon.png` a small icon that browsers can use when referencing the website - `Resources` - contains the current `Interlisp-D` logo, used on the home page, and another instance of `favicon.png` - - `CNAME` - a oneline text file that provides support for using a [custom domain](https://gohugo.io/hosting-and-deployment/hosting-on-github/#use-a-custom-domain) + - `CNAME` - a one line text file that provides support for using a [custom domain](https://gohugo.io/hosting-and-deployment/hosting-on-github/#use-a-custom-domain) ## Search @@ -256,7 +338,7 @@ layout as being `search`. ### Updating Search -Modfying the websites that are searched requires updating the Google Custom +Modifying the websites that are searched requires updating the Google Custom Search engine settings. This is done via logging into Google's Programmable Search Engine Dashboard at: [https://programmablesearchengine.google.com](https://programmablesearchengine.google.com) diff --git a/config/_default/privacy.yaml b/config/_default/privacy.yaml index 0c4f9e895..0dc6a6395 100644 --- a/config/_default/privacy.yaml +++ b/config/_default/privacy.yaml @@ -5,7 +5,7 @@ # See: https://gohugo.io/about/hugo-and-gdpr/ # # googleAnalytics: -# ananymizeIP: Enable anonymization of IP addresses +# anonymizeIP: Enable anonymization of IP addresses # disable: Set to true to disable googleAnalytics # respectDoNotTrack: Check for Do Not Track in headers # useSessionStorage: Store session information in storage and