Jenkinsfile-redo-solr

pipeline {
    agent { label 'monarch-agent-large' }
    environment {
        HOME = "${env.WORKSPACE}"
        PATH = "/opt/poetry/bin:${env.PATH}"
        AWS_ACCESS_KEY_ID = credentials('AWS_ACCESS_KEY_ID')        
        AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
    }
    stages {
        stage('Setup parameters') {
            steps {
                script {
                    properties([
                        parameters([
                            stringParam(
                                defaultValue: 'latest',
                                description: 'monarch-kg-dev release',
                                name: 'RELEASE'
                            ),
                            booleanParam(
                                defaultValue: true,
                                description: 'Run KGX Graph Summary',
                                name: 'KGX_GRAPH_SUMMARY'
                            ),
                            booleanParam(
                                defaultValue: true,
                                description: 'Re-run denormalization step',
                                name: 'RUN_CLOSURIZER'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Load Solr',
                                name: 'SOLR'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Load sqlite',
                                name: 'SQLITE'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Make TSV Exports',
                                name: 'EXPORT'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Make JSONL Export',
                                name: 'JSONL'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Run KGX Transforms',
                                name: 'KGX_TRANSFORMS'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Upload to bucket',
                                name: 'UPLOAD'
                            ),
                            booleanParam(
                                defaultValue: false,
                                description: 'Update dev deployment to latest release',
                                name: 'UPDATE_DEV'
                            )
                        ])
                    ])
                }
            }
        }
        stage('setup') {
            steps {
                sh '''
                    echo "Current directory: $(pwd)"
                    # export PATH=$PATH:$HOME/.local/bin
                    echo "Path: $PATH"
                    
                    # echo $SHELL
                    python3 --version
                    pip --version
                    poetry --version
                  

                    # poetry config experimental.new-installer false
                    poetry install --with dev
                    poetry run which ingest

                '''
            }
        }
        stage('download kg artifacts') {
            steps {
                sh '''
                    mkdir output || true
                    mkdir data || true
                    gsutil -q -m cp -r gs://data-public-monarchinitiative/monarch-kg-dev/${env.RELEASE}/* output/
                    gsutil -q -m cp -r gs://monarch-ingest-data-cache/* data/
                    ls -lasdf
                '''
            }
        }
        stage('kgx-graph-summary') {
            when {
                expression {
                   return params.KGX_GRAPH_SUMMARY
                }
            }
            steps {
                sh 'poetry run kgx graph-summary -i tsv -c "tar.gz" --node-facet-properties provided_by --edge-facet-properties provided_by output/monarch-kg.tar.gz -o output/merged_graph_stats.yaml'
            }
        }
        stage('denormalize') {
            when {
                expression {
                   return params.RUN_CLOSURIZER
                }
            }
            steps {
                sh 'poetry run ingest closure'
            }
        }
        stage('solr') {
            when {
                expression {
                   return params.SOLR
                }
            }
            steps {
                sh 'poetry run ingest solr'
            }
        }
        stage('sqlite') {
            when {
                expression {
                   return params.SQLITE
                }
            }
            steps {
                sh 'poetry run ingest sqlite'
            }
        }
        stage('make tsv exports') {
            when {
                expression {
                   return params.EXPORT
                }
            }
            steps {
                sh 'poetry run ingest export'
            }
        }
        stage("jsonl") {
            when {
                expression {
                   return params.JSONL
               }
            }
           steps {
               sh 'poetry run ingest jsonl'
           }
        }
        stage('kgx-transforms'){
            when {
               expression {
                   return params.KGX_TRANSFORMS
                   }
            }
            steps {
                sh './scripts/kgx_transforms.sh'
            }
        }
        stage('upload files') {
            when {
                expression {
                   return params.UPLOAD
                }
            }
            steps {
                sh 'scripts/update_latest_release.sh'
            }
        }
        stage('update dev deployment') {
            when {
                expression {
                   return params.UPDATE_DEV
                }
            }
            steps {
                sh 'poetry run python scripts/update-dev-solr.py'
            }
        }
    }
}