Job request: 2687

######################################

# This script defines the project pipeline - it specifys the execution orders for all the code in this
# repo using a series of actions.

######################################

version: '3.0'

expectations:
  population_size: 100000

actions:

  # Extract data ----
  
  ## Cohort data
  generate_cohort_1:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2019-01-01 to 2019-06-01 by month" --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/input_*.feather
        
  generate_cohort_2:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2019-07-01 to 2019-12-01 by month" --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/input*.feather
  
  generate_cohort_3:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2020-01-01 to 2020-06-01 by month" --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/inpu*.feather
        
  generate_cohort_4:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2020-07-01 to 2020-12-01 by month" --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/inp*.feather
        
  generate_cohort_5:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2021-01-01 to 2021-04-01 by month" --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/in*.feather
  
  ## Ethnicity      
  extract_ethnicity:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_ethnicity --output-dir=output/data --output-format=feather
    outputs:
      highly_sensitive:
        cohort: output/data/input_ethnicity.feather
        
  # ## Patient to practice lookup
  # patient_practice_lookup:
  #   run: cohortextractor:latest generate_cohort --study-definition study_definition_practice_count --output-dir=output/data --output-format=feather
  #   outputs:
  #     highly_sensitive:
  #       cohort: output/data/input_practice_count.feather
        
  
  # Data processing ----

  # ## Patient counts
  # get_patient_count:
  #   run: python:latest python analysis/get_patient_counts.py --output-dir=output/data
  #   needs: [extract_data]
  #   outputs:
  #      moderately_sensitive:
  #        text: output/data/patient_count.json
         
  # ## Practice count
  # get_practice_count:
  #   run: python:latest python analysis/get_practice_count.py --output-dir=output/data
  #   needs: [patient_practice_lookup]
  #   outputs:
  #     moderately_sensitive:
  #       text: output/data/practice_count.json
  
  ## Add ethnicity
  join_ethnicity:
    run: python:latest python analysis/join_ethnicity.py
    needs: [generate_cohort_1, generate_cohort_2, generate_cohort_3, generate_cohort_4, generate_cohort_5, extract_ethnicity]
    outputs:
      highly_sensitive:
        cohort: output/data/i*.feather
  
  ## Process data
  data_process:
    run: r:latest analysis/process_data.R
    needs: [join_ethnicity]
    outputs:
      moderately_sensitive:
        data: output/data/data_*.rds
        # tables: output/data/custom_measures_*.csv
        
  # ## Generate measures
  # generate_measures:
  #     run: cohortextractor:latest generate_measures --study-definition study_definition --output-dir=output/data
  #     needs: [join_ethnicity]
  #     outputs:
  #       moderately_sensitive:
  #         measure_csv: output/data/measure_*.csv
  
  # ### Generate measures, by group
  # generate_measures_demographics:
  #   run: python:latest python analysis/calculate_measures.py
  #   needs: [join_ethnicity]
  #   outputs:
  #     moderately_sensitive:
  #       measure: output/combined_measure_*.csv
  # 
  #   ### Generate measures, by group and demographic
  # generate_measures_demographics:
  #   run: python:latest python analysis/calculate_measures.py
  #   needs: [join_ethnicity]
  #   outputs:
  #     moderately_sensitive:
  #       measure: output/combined_measure_*.csv
  
  

  # Results ----
  
  ## Figures
  summary_figures:
    run: r:latest analysis/summary_plots.R
    needs: [data_process]
    outputs:
      moderately_sensitive:
        plots: output/figures/plot_*.svg
  
  summary_figures_redacted:
    run: r:latest analysis/summary_plots_redacted.R
    needs: [data_process]
    outputs:
      moderately_sensitive:
        plots: output/figures/plot*.svg
        
  ## Table 1
  table_1:
    run: r:latest analysis/table_1.R
    needs: [join_ethnicity]
    outputs:
      moderately_sensitive:
        plots: output/tables/table1.html
  
  # ## Whole population notebook
  # generate_notebook:
  #   run: jupyter:latest jupyter nbconvert /workspace/notebooks/antipsychotic_measures.ipynb --execute --to html --template basic --output-dir=/workspace/output --ExecutePreprocessor.timeout=86400 --no-input
  #   needs: [generate_measures, get_practice_count, get_patient_count]
  #   outputs:
  #     moderately_sensitive:
  #       notebook: output/antipsychotic_measures.html