Fixed few subtle threading bugs uncovered by stress test and rewrote the test to be model-based (#1157)

* testing: refactored logs directory management

* content: fixed index mutex to be shared across all write sessions

added mutex protection during writecontent/refresh race

* testing: upload log artifacts

* content: bump revision number after index has been added

This fixes a bug where manifest manager in another session for
the same open repository may not see a content added, because they
will prematurely cache the incomplete set of contents.

This took 2 weeks to find.

* manifest: improved log output, fixed unnecessary mutex release

* testing: rewrote stress test to be model-based and more precise

.github/workflows/code-coverage.yml

@@ -18,3 +18,10 @@ jobs:
       run: make test-with-coverage
     - name: Publish Coverage Results
       run: make ci-publish-coverage
+    - name: Upload Logs
+      uses: actions/upload-artifact@v2
+      with:
+        name: logs
+        path: .logs/**/*.log
+        if-no-files-found: ignore
+      if: ${{ always() }}

.github/workflows/make.yml

@@ -103,6 +103,13 @@ jobs:
     - name: Integration Tests
       run: make -j2 ci-integration-tests
       continue-on-error: ${{ github.event_name != 'pull_request' }}
+    - name: Upload Logs
+      uses: actions/upload-artifact@v2
+      with:
+        name: logs
+        path: .logs/**/*.log
+        if-no-files-found: ignore
+      if: ${{ always() }}
     - name: Upload Kopia Artifacts
       uses: actions/upload-artifact@v2
       with:

.github/workflows/stress-test.yml

@@ -0,0 +1,33 @@
+name: Stress Test
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - v*
+  pull_request: {}
+  schedule:
+    # run every 2 hours
+    - cron:  '12 */2 * * *'
+jobs:
+  endurance-test:
+    name: Stress Test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go.
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.16
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Stress Test
+      run: make stress-test
+    - name: Upload Logs
+      uses: actions/upload-artifact@v2
+      with:
+        name: logs
+        path: .logs/**/*.log
+        if-no-files-found: ignore
+      if: ${{ always() }}