VFB_reporting_results/.github/workflows/create_reports.yml at master · VirtualFlyBrain/VFB_reporting_results · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: Create Reports
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  notebooks:
    name: "Generate Reports"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Pull Code
        run: |
          git clone https://github.com/VirtualFlyBrain/VFB_reporting.git
          git clone https://github.com/VirtualFlyBrain/VFB_neo4j.git --single-branch
          ln -s ../ ./VFB_reporting/VFB_reporting_results

      - name: Install dependencies
        run: |
          python -m pip install -U pip
          python -m pip install -r VFB_reporting/requirements.txt
          python -m pip install -r VFB_neo4j/requirements.txt

      - name: Run query tools test
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./test/query_tools_test.py

      - name: Run report runner
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./report_runner.py

      - name: Run comparison
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./comparison.py

      - name: Run get catmaid cellTypes
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./get_catmaid_cellTypes.py

      - name: Run get catmaid papers
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./get_catmaid_papers.py

      - name: Run VFB content report generator
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./VFB_content_report_generator.py

      - name: Run ID mapping tables
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./ID_mapping_tables.py

      - name: Run individual classification report
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./individual_classification_report.py

      - name: Run EM instance-FBbt conflict report
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./Instance_FBbt_conflict_report.py

      - name: Run connectome multiple symbol-bearing class report
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./connectome_multi_symbol_class_report.py

      - name: Run scRNAseq DataSet report
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          python ./scRNAseq_datasets.py

      - name: Create curation files
        run: |
          export PYTHONPATH=$PYTHONPATH:$PWD/VFB_neo4j/src/
          cd VFB_reporting/src
          rm -vf ../VFB_reporting_results/anat_*.tsv
          rm -vf ../VFB_reporting_results/anat_*.yaml
          ls -lh ../VFB_reporting_results/
          python ./make_curation_records/anat_curation_file_maker.py

      - name: Regenerate README
        run: |
          cat README_header.md VFB_reporting/reports.md > README.md

      - name: Run GA4 usage reports
        # Aggregate counts only. Needs the GA4_SERVICE_ACCOUNT_KEY_JSON repository secret
        # (Viewer on property 375054915); skipped without it, and never blocks the other reports.
        continue-on-error: true
        env:
          GA4_SERVICE_ACCOUNT_KEY_JSON: ${{ secrets.GA4_SERVICE_ACCOUNT_KEY_JSON }}
        run: |
          if [ -z "$GA4_SERVICE_ACCOUNT_KEY_JSON" ]; then echo "No GA4 secret set; skipping"; exit 0; fi
          cd VFB_reporting/src
          python ./ga4_usage_collect.py
          python ./ga4_usage_summary.py

      - name: Clean up
        run: |
          rm -rfv VFB_reporting
          rm -rfv VFB_neo4j

      - name: On Completion
        run: |
          git config --global user.email "support@virtualflybrain.org"
          git config --global user.name "gitHub actions"
          git add .
          git status

      - name: Commit changes
        uses: EndBug/add-and-commit@v7
        with:
          author_name: GitHub Actions
          author_email: support@virtualflybrain.org
          message: 'Automated report run'
          add: '["*.tsv","*.md --force"]'