Merge pull request #15 from actions/jess/compare-outputs-in-ci

Compare jekyll build output in CI
actions · Feb 7, 2022 · 57502bc · 57502bc
2 parents c96882a + 5a78e45
commit 57502bc
Show file tree

Hide file tree

Showing 76 changed files with 7,803 additions and 29 deletions.
diff --git a/.github/workflows/record.yml b/.github/workflows/record.yml
@@ -0,0 +1,68 @@
+name: Record Expected
+
+on:
+  workflow_dispatch:
+
+jobs:
+  record-expected-output:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Build test project 'simple'
+        uses: ./
+        with:
+          source: test_projects/simple
+          destination: test_projects/simple/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'readme'
+        uses: ./
+        with:
+          source: test_projects/readme
+          destination: test_projects/readme/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'octicons'
+        uses: ./
+        with:
+          source: test_projects/octicons
+          destination: test_projects/octicons/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'mojombo'
+        uses: ./
+        with:
+          source: test_projects/mojombo
+          destination: test_projects/mojombo/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'themes'
+        uses: ./
+        with:
+          source: test_projects/themes
+          destination: test_projects/themes/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'jekyll-include-cache'
+        uses: ./
+        with:
+          source: test_projects/jekyll-include-cache
+          destination: test_projects/jekyll-include-cache/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'future-false'
+        uses: ./
+        with:
+          source: test_projects/future-false
+          destination: test_projects/future-false/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
+      - name: Build test project 'future-true'
+        uses: ./
+        with:
+          source: test_projects/future-true
+          destination: test_projects/future-true/_expected
+          build_revision: JEKYLL_BUILD_REVISION
+
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -28,13 +28,12 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-readme:
     runs-on: ubuntu-latest
@@ -53,13 +52,12 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-octicons:
     runs-on: ubuntu-latest
@@ -78,13 +76,12 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-mojombo:
     runs-on: ubuntu-latest
@@ -103,13 +100,12 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-themes:
     runs-on: ubuntu-latest
@@ -128,13 +124,12 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-jekyll-include-cache:
     runs-on: ubuntu-latest
@@ -153,11 +148,57 @@ jobs:
         with:
           source: ./test_projects/${{ env.TEST_NAME }}
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
+
+  test-future-false:
+    runs-on: ubuntu-latest
+    env:
+      TEST_NAME: future-false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Render Action Metadata
+        run: |
+          ./bin/render_action_metadata Dockerfile > action.yml
+
+      - name: Test ${{ env.TEST_NAME }} Project
+        uses: ./
         with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+          source: ./test_projects/${{ env.TEST_NAME }}
+          destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
+
+  test-future-true:
+    runs-on: ubuntu-latest
+    env:
+      TEST_NAME: future-true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
 
+      - name: Render Action Metadata
+        run: |
+          ./bin/render_action_metadata Dockerfile > action.yml
+
+      - name: Test ${{ env.TEST_NAME }} Project
+        uses: ./
+        with:
+          source: ./test_projects/${{ env.TEST_NAME }}
+          destination: ./test_projects/${{ env.TEST_NAME }}/_site
+          build_revision: JEKYLL_BUILD_REVISION
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
diff --git a/bin/compare_expected_output b/bin/compare_expected_output
@@ -0,0 +1,35 @@
+#! /usr/bin/env ruby
+
+require "pathname"
+require "shellwords"
+
+project_path = ARGV[0]
+
+Dir.chdir(project_path)
+
+expected_files = Dir["_expected/**/*"].select { |path| File.file?(path) }.map { |path| Pathname.new(path).relative_path_from("_expected").to_s }
+actual_files = Dir["_site/**/*"].select { |path| File.file?(path) }.map { |path| Pathname.new(path).relative_path_from("_site").to_s }
+
+differences = []
+
+expected_files.each do |expected_file|
+  if actual_files.include?(expected_file)
+    diff = `diff #{Shellwords.escape(File.join("_expected", expected_file))} #{Shellwords.escape(File.join("_site", expected_file))}`
+    if !$?.success?
+      differences << "Expected output of #{expected_file} differs:\n#{diff}"
+    end
+  else
+    differences << "Missing expected file: #{expected_file}"
+  end
+end
+
+unexpected_files = actual_files - expected_files
+unexpected_files.each do |unexpected_file|
+  differences << "Unexpected file: #{unexpected_file}"
+end
+
+if !differences.empty?
+  STDERR.puts "Differences between expected and actual outputs:"
+  differences.each { |diff| STDERR.puts(diff) }
+  exit(1)
+end
diff --git a/bin/record_expected_output b/bin/record_expected_output
@@ -0,0 +1,14 @@
+#! /usr/bin/env sh
+
+if [ "local" = "$1" ]; then
+	export JEKYLL_ENV=production
+	export PAGES_REPO_NWO=actions/jekyll-build-pages
+	export JEKYLL_BUILD_REVISION=JEKYLL_BUILD_REVISION
+
+	for dir in $(ls test_projects)
+	do
+		bundle exec github-pages build --verbose -s test_projects/$dir -d test_projects/$dir/_expected
+	done
+else
+	act -b -s GITHUB_TOKEN -j record-expected-output
+fi
diff --git a/test_projects/future-false/_config.yaml b/test_projects/future-false/_config.yaml
@@ -0,0 +1,2 @@
+time: 2022-02-02 02:02:02
+timezone: America/New_York
diff --git a/test_projects/future-false/_expected/2020/02/13/the-future-is-looking-bright.html b/test_projects/future-false/_expected/2020/02/13/the-future-is-looking-bright.html
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+<!-- Begin Jekyll SEO tag v2.7.1 -->
+<title>The Future is Looking Bright! | jekyll-build-pages</title>
+<meta name="generator" content="Jekyll v3.9.0" />
+<meta property="og:title" content="The Future is Looking Bright!" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Everything’s coming up Milhouse." />
+<meta property="og:description" content="Everything’s coming up Milhouse." />
+<link rel="canonical" href="https://github.com/pages/actions/jekyll-build-pages/2020/02/13/the-future-is-looking-bright.html" />
+<meta property="og:url" content="https://github.com/pages/actions/jekyll-build-pages/2020/02/13/the-future-is-looking-bright.html" />
+<meta property="og:site_name" content="jekyll-build-pages" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2020-02-13T00:00:00-05:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="The Future is Looking Bright!" />
+<script type="application/ld+json">
+{"mainEntityOfPage":{"@type":"WebPage","@id":"https://github.com/pages/actions/jekyll-build-pages/2020/02/13/the-future-is-looking-bright.html"},"description":"Everything’s coming up Milhouse.","url":"https://github.com/pages/actions/jekyll-build-pages/2020/02/13/the-future-is-looking-bright.html","@type":"BlogPosting","headline":"The Future is Looking Bright!","dateModified":"2020-02-13T00:00:00-05:00","datePublished":"2020-02-13T00:00:00-05:00","@context":"https://schema.org"}</script>
+<!-- End Jekyll SEO tag -->
+
+    <link rel="stylesheet" href="/pages/actions/jekyll-build-pages/assets/css/style.css?v=JEKYLL_BUILD_REVISION">
+    <!-- start custom head snippets, customize with your own _includes/head-custom.html file -->
+
+<!-- Setup Google Analytics -->
+
+
+
+<!-- You can set your favicon here -->
+<!-- link rel="shortcut icon" type="image/x-icon" href="/pages/actions/jekyll-build-pages/favicon.ico" -->
+
+<!-- end custom head snippets -->
+
+  </head>
+  <body>
+    <div class="container-lg px-3 my-5 markdown-body">
+
+      <h1><a href="https://github.com/pages/actions/jekyll-build-pages/">jekyll-build-pages</a></h1>
+
+
+      <p>Everything’s coming up Milhouse.</p>
+
+
+
+      <div class="footer border-top border-gray-light mt-5 pt-3 text-right text-gray">
+        This site is open source. <a href="https://github.com/actions/jekyll-build-pages/edit/gh-pages/_posts/2020-02-13-the-future-is-looking-bright.md">Improve this page</a>.
+      </div>
+
+    </div>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.1.0/anchor.min.js" integrity="sha256-lZaRhKri35AyJSypXXs4o6OPFTbTmUoltBbDCbdzegg=" crossorigin="anonymous"></script>
+    <script>anchors.add();</script>
+  </body>
+</html>
diff --git a/test_projects/future-false/_expected/assets/css/style.css b/test_projects/future-false/_expected/assets/css/style.css
diff --git a/test_projects/future-false/_posts/2020-02-13-the-future-is-looking-bright.md b/test_projects/future-false/_posts/2020-02-13-the-future-is-looking-bright.md
@@ -0,0 +1,5 @@
+---
+title: The Future is Looking Bright!
+---
+
+Everything's coming up Milhouse.
diff --git a/test_projects/future-false/_posts/2022-03-01-we-live-in-hell.md b/test_projects/future-false/_posts/2022-03-01-we-live-in-hell.md
@@ -0,0 +1,5 @@
+---
+title: We Live In Hell
+---
+
+And it's not even the cool hell that \[redacted\] are afraid of. 😭
diff --git a/test_projects/future-true/_config.yaml b/test_projects/future-true/_config.yaml
@@ -0,0 +1,3 @@
+time: 2022-02-02 02:02:02
+timezone: America/New_York
+future: true