Add a test step to compare outputs

There is no expected output committed to the repo yet, so it is expected
to fail.

.github/workflows/test.yml

@@ -30,11 +30,9 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-readme:
     runs-on: ubuntu-latest
@@ -55,11 +53,9 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-octicons:
     runs-on: ubuntu-latest
@@ -80,11 +76,9 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-mojombo:
     runs-on: ubuntu-latest
@@ -105,11 +99,9 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-themes:
     runs-on: ubuntu-latest
@@ -130,11 +122,9 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 
   test-jekyll-include-cache:
     runs-on: ubuntu-latest
@@ -155,9 +145,7 @@ jobs:
           destination: ./test_projects/${{ env.TEST_NAME }}/_site
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Save test results
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: ./test_projects/${{ env.TEST_NAME }}
+      - name: Verify output
+        run: |
+          ./bin/compare_expected_output ./test_projects/${{env.TEST_NAME}}
 

bin/compare_expected_output

@@ -0,0 +1,36 @@
+#! /usr/bin/env ruby
+
+require "pathname"
+require "shellwords"
+
+project_path = ARGV[0]
+
+Dir.chdir(project_path)
+
+expected_files = Dir["_expected/**/*"].map { |path| Pathname.new(path).relative_path_from("_expected").to_s }
+actual_files = Dir["_site/**/*"].map { |path| Pathname.new(path).relative_path_from("_site").to_s }
+
+differences = []
+
+expected_files.each do |expected_file|
+  if actual_files.include?(expected_file)
+    # TODO: consider -b to ignore whitespace, or -B for ignore blank lines, or --strip-trailing-cr
+    diff = `diff #{Shellwords.escape(File.join("_expected", expected_file))} #{Shellwords.escape(File.join("_site", expected_file))}`
+    if !$?.success?
+      differences << "Expected output of #{expected_file} differs:\n#{diff}"
+    end
+  else
+    differences << "Missing expected file: #{expected_file}"
+  end
+end
+
+unexpected_files = actual_files - expected_files
+unexpected_files.each do |unexpected_file|
+  differences << "Unexpected file: #{unexpected_file}"
+end
+
+if !differences.empty?
+  STDERR.puts "Differences between expected and actual outputs:"
+  differences.each { |diff| STDERR.puts(diff) }
+  exit(1)
+end