RFC: Initial LIT-based integration tests setup

Makefile

@@ -9,7 +9,7 @@ CONFIG := $(wildcard *.py)
 MODULES := $(wildcard $(PACKAGE)/*.py)
 
 # Virtual environment paths
-VIRTUAL_ENV ?= .venv
+VIRTUAL_ENV ?= $(PWD)/.venv
 
 # MAIN TASKS ##################################################################
 
@@ -83,7 +83,7 @@ NOSE_OPTIONS += --with-cov --cov=$(PACKAGE) --cov-report=html --cov-report=term-
 endif
 
 .PHONY: test
-test: test-all ## Run unit and integration tests
+test: test-all test-lit ## Run unit and integration tests
 
 .PHONY: test-unit
 test-unit: install
@@ -93,6 +93,16 @@ test-unit: install
 .PHONY: test-int
 test-int: test-all
 
+.PHONY: test-lit
+DOORSTOP_EXEC=$(VIRTUAL_ENV)/bin/doorstop
+test-lit: install ## Run LIT integration tests
+	cd tests/integration && make clean
+	CURRENT_DIR=$(PWD) \
+		DOORSTOP_EXEC=$(DOORSTOP_EXEC) \
+		PATH=$(PWD)/tests/integration/tools/FileCheck:$(PWD)/tests/integration/tools:$$PATH \
+		poetry run lit \
+		-vv $(PWD)/tests/integration
+
 .PHONY: test-all
 test-all: install
 	TEST_INTEGRATION=true poetry run nosetests $(PACKAGES) $(NOSE_OPTIONS) --show-skipped

pyproject.toml

@@ -69,6 +69,7 @@ nose-cov = "*"
 nose-capturestderr = "*"
 nose-show-skipped = "*"
 expecter = "*"
+lit = "*"
 
 # Reports
 coveragespace = "*"

tests/integration/.gitignore

@@ -0,0 +1,7 @@
+# Files produced by LLVM LIT
+**/Output/**
+*.script
+
+# Files produced by the test cases
+**/Sandbox/**
+

tests/integration/Makefile

@@ -0,0 +1,24 @@
+# Self-Documented Makefile
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+.PHONY: help
+help: ## Show this help message.
+	@grep -E '^[a-zA-Z0-9_\.-\%]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+CLEAN_FIND_CMD=find \
+	. \
+	-type f \( \
+		-name '*.script' \
+	\) \
+	-or -type d \( \
+		-name '*.dSYM' -or \
+		-name 'Sandbox' -or \
+		-name 'Output' \
+	\) \
+	-not -path "**Expected**" \
+	-not -path "**Input**"
+
+CLEAN_FILES=$(shell $(CLEAN_FIND_CMD))
+clean: ## Clean all temporary artefacts
+	echo $(CLEAN_FILES) | \
+		xargs rm -rfv
+

tests/integration/lit.cfg

@@ -0,0 +1,15 @@
+import lit.formats
+
+config.name = "Doorstop integration tests"
+config.test_format = lit.formats.ShTest("0")
+
+# LLVM LIT: Is it possible to pass user defined parameter into a test script?
+# https://stackoverflow.com/a/39861848/598057
+current_dir = os.environ.get('CURRENT_DIR', '')
+doorstop_exec = os.environ.get('DOORSTOP_EXEC', '')
+
+config.substitutions.append(('%CURRENT_DIR', current_dir))
+config.substitutions.append(('%DOORSTOP_EXEC', doorstop_exec))
+config.substitutions.append(('%COMPARE_EXEC', "{}/tests/integration/tools/compare_dirs".format(current_dir)))
+
+config.suffixes = ['.itest']

tests/integration/tests/basic_functionality/create/01_creating_parent/Expected/.doorstop.yml

@@ -0,0 +1,4 @@
+settings:
+  digits: 3
+  prefix: TST
+  sep: ''

tests/integration/tests/basic_functionality/create/01_creating_parent/sample.itest

@@ -0,0 +1,7 @@
+; RUN: rm -rf %S/Sandbox
+; RUN: /bin/mkdir -p %S/Sandbox && cd %S/Sandbox
+; RUN: git init .
+; RUN: %DOORSTOP_EXEC create TST . | FileCheck %s
+; RUN: %COMPARE_EXEC %S/Sandbox %S/Expected
+; CHECK: building tree...
+; CHECK-NEXT: created document: TST (@/.)

tests/integration/tests/basic_functionality/create/02_creating_item/Expected/.doorstop.yml

@@ -0,0 +1,4 @@
+settings:
+  digits: 3
+  prefix: TST
+  sep: ''

tests/integration/tests/basic_functionality/create/02_creating_item/Expected/TST001.yml

@@ -0,0 +1,9 @@
+active: true
+derived: false
+header: ''
+level: 1.0
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: ''

tests/integration/tests/basic_functionality/create/02_creating_item/sample.itest

@@ -0,0 +1,8 @@
+; RUN: rm -rf %S/Sandbox
+; RUN: /bin/mkdir -p %S/Sandbox && cd %S/Sandbox
+; RUN: git init .
+; RUN: %DOORSTOP_EXEC create TST .
+; RUN: %DOORSTOP_EXEC add TST | FileCheck %s
+; RUN: %COMPARE_EXEC %S/Sandbox %S/Expected
+; CHECK: building tree...
+; CHECK-NEXT: added item: TST001 (@/TST001.yml)

tests/integration/tests/basic_functionality/create/03_creating_3_items/Expected/.doorstop.yml

@@ -0,0 +1,4 @@
+settings:
+  digits: 3
+  prefix: TST
+  sep: ''

tests/integration/tests/basic_functionality/create/03_creating_3_items/Expected/TST001.yml

@@ -0,0 +1,9 @@
+active: true
+derived: false
+header: ''
+level: 1.0
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: ''

tests/integration/tests/basic_functionality/create/03_creating_3_items/Expected/TST002.yml

@@ -0,0 +1,9 @@
+active: true
+derived: false
+header: ''
+level: 1.1
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: ''

tests/integration/tests/basic_functionality/create/03_creating_3_items/Expected/TST003.yml

@@ -0,0 +1,9 @@
+active: true
+derived: false
+header: ''
+level: 1.2
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: ''

tests/integration/tests/basic_functionality/create/03_creating_3_items/Input/.doorstop.yml

@@ -0,0 +1,4 @@
+settings:
+  digits: 3
+  prefix: TST
+  sep: ''

tests/integration/tests/basic_functionality/create/03_creating_3_items/sample.itest

@@ -0,0 +1,14 @@
+; RUN: rm -rf %S/Sandbox
+; RUN: cp -r %S/Input %S/Sandbox && cd %S/Sandbox
+; RUN: git init .
+; RUN: %DOORSTOP_EXEC add TST \
+; RUN:   | FileCheck %s -check-prefix=CHECK-COMMAND1
+; RUN: %DOORSTOP_EXEC add TST \
+; RUN:   | FileCheck %s -check-prefix=CHECK-COMMAND2
+; RUN: %DOORSTOP_EXEC add TST \
+; RUN:   | FileCheck %s -check-prefix=CHECK-COMMAND3
+; RUN: %COMPARE_EXEC %S/Sandbox %S/Expected
+
+; CHECK-COMMAND1: added item: TST001 (@/TST001.yml)
+; CHECK-COMMAND2: added item: TST002 (@/TST002.yml)
+; CHECK-COMMAND3: added item: TST003 (@/TST003.yml)

tests/integration/tests/basic_functionality/no_command/01_empty_folder/sample.itest

@@ -0,0 +1,11 @@
+; RUN: rm -rf %S/Sandbox
+; RUN: /bin/mkdir -p %S/Sandbox && cd %S/Sandbox
+;
+; Note: We must pass -j %S/Sandbox otherwise Doorstop walks up and finds
+; irrelevant files from the reqs folder outside of tests/ folder.
+; RUN: %DOORSTOP_EXEC -j %S/Sandbox 2>&1 | FileCheck %s
+;
+; CHECK: building tree...
+; CHECK-NEXT: loading documents...
+; CHECK-NEXT: validating items...
+; CHECK-NEXT: WARNING: no documents

tests/integration/tests/basic_functionality/no_command/02_empty_folder_with_git_init/sample.itest

@@ -0,0 +1,8 @@
+; RUN: rm -rf %S/Sandbox
+; RUN: /bin/mkdir -p %S/Sandbox && cd %S/Sandbox
+; RUN: git init .
+; RUN: %DOORSTOP_EXEC -j %S/Sandbox 2>&1 | FileCheck %s
+; CHECK: building tree...
+; CHECK-NEXT: loading documents...
+; CHECK-NEXT: validating items...
+; CHECK-NEXT: WARNING: no documents

tests/integration/tools/FileCheck/FileCheck

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+file_check_path=`echo $(cd $(dirname "$0") && pwd -P)`
+
+echo $file_check_path
+
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+  $file_check_path/FileCheck-Linux "$@"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  $file_check_path/FileCheck-macOS "$@"
+else
+  exit 1
+fi