Skip to content

Commit

Permalink
add code scan function (#21)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WenjiaoYue
WenjiaoYue committed Aug 16, 2022
1 parent b7ede92 commit c1101be
Show file tree
Hide file tree
Showing 6 changed files with 157 additions and 200 deletions.
302 changes: 143 additions & 159 deletions .azure-pipelines/code-scan.yml
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
trigger: none
pr:
- master

pool: suyue-test

variables:
IMAGE_NAME: "code-scan"
IMAGE_TAG: "1.0"
IMAGE_NAME: "neural-compressor"
IMAGE_TAG: "py38"
CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan"
CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"
CURRENT_PATH: $(Build.SourcesDirectory)
CURRENT_STATUS: true
ER: "1212"

stages:
- stage: BanditCodeScan
Expand All @@ -19,7 +18,6 @@ stages:
jobs:
- job: Bandit
displayName: Bandit
condition: eq(variables['CURRENT_STATUS'], 'true')
steps:
- script: |
echo ${BUILD_SOURCESDIRECTORY}
Expand Down Expand Up @@ -47,164 +45,150 @@ stages:
inputs:
targetType: "inline"
script: |
docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS)
echo "------$(CURRENT_STATUS)-------"
echo $(CURRENT_STATUS)
docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh
echo "---$(ER)----$(QR)------"
displayName: "Bandit Check"

- task: PublishPipelineArtifact@1
condition: always()
inputs:
targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log
artifact: bandit
publishLocation: "pipeline"
displayName: "PublishPipelineArtifact"


- stage: PylintCodeScan
displayName: Pylint Code Scan
dependsOn: []
jobs:
- job: Pylint
displayName: Pylint
steps:
- script: |
echo ${BUILD_SOURCESDIRECTORY}
sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
echo y | docker system prune
displayName: "Clean workspace"
- checkout: self
displayName: "Checkout out Repo"
- script: |
if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
fi
docker images | grep -i ${IMAGE_NAME}
if [[ $? -ne 0 ]]; then
echo "NO Such Image ${IMAGE_NAME}"
exit 1
fi
displayName: "Build Devel Images"
- script: |
docker stop $(docker ps -aq)
docker rm -vf $(docker ps -aq) || true
displayName: "Clean Docker"
- task: Bash@3
inputs:
targetType: "inline"
script: |
docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh
displayName: "Pylint Check"

- task: PublishPipelineArtifact@1
condition: always()
inputs:
targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json
artifact: pylint
publishLocation: "pipeline"
displayName: "Status Check"


# - script: |
# if [ $(CURRENT_STATUS) -ne 0 ] ; then
# exit 1
# fi
# exit 0



# - stage: PylintCodeScan
# displayName: Pylint Code Scan
# dependsOn: []
# jobs:
# - job: Pylint
# displayName: Pylint
# steps:
# - script: |
# if [ $(CURRENT_STATUS) -ne 0 ] ; then
# exit 1
# fi
# exit 0
# - script: |
# echo ${BUILD_SOURCESDIRECTORY}
# sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
# echo y | docker system prune
# displayName: "Clean workspace"
# - checkout: self
# displayName: "Checkout out Repo"
# - script: |
# if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
# docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
# fi
# docker images | grep -i ${IMAGE_NAME}
# if [[ $? -ne 0 ]]; then
# echo "NO Such Image ${IMAGE_NAME}"
# exit 1
# fi
# displayName: "Build Devel Images"
# - script: |
# docker stop $(docker ps -aq)
# docker rm -vf $(docker ps -aq) || true
# displayName: "Clean Docker"

# - task: Bash@3
# inputs:
# targetType: "inline"
# script: |
# docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh

# - task: PublishPipelineArtifact@1
# inputs:
# targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json
# artifact: pylint
# publishLocation: "pipeline"

# - script: |
# if [ $(CURRENT_STATUS) -ne 0 ] ; then
# exit 1
# fi
# exit 0

# - stage: PyspellingCodeScan
# displayName: Pyspelling Code Scan
# dependsOn: []
# jobs:
# - job: Pyspelling
# displayName: Pyspelling
# steps:
# - script: |
# if [ $(CURRENT_STATUS) -ne 0 ] ; then
# exit 1
# fi
# exit 0
# - script: |
# echo ${BUILD_SOURCESDIRECTORY}
# sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
# echo y | docker system prune
# displayName: "Clean workspace"
# - checkout: self
# displayName: "Checkout out Repo"
# - script: |
# if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
# docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
# fi
# docker images | grep -i ${IMAGE_NAME}
# if [[ $? -ne 0 ]]; then
# echo "NO Such Image ${IMAGE_NAME}"
# exit 1
# fi
# displayName: "Build Devel Images"
# - script: |
# docker stop $(docker ps -aq)
# docker rm -vf $(docker ps -aq) || true
# displayName: "Clean Docker"

# - task: Bash@3
# inputs:
# targetType: "inline"
# script: |
# docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh

# - task: PublishPipelineArtifact@1
# inputs:
# targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log
# artifact: pyspelling
# publishLocation: "pipeline"

# - stage: CopyRight
# displayName: CopyRight Code Scan
# dependsOn: []
# jobs:
# - job: CopyRight
# displayName: CopyRight
# steps:
# - task: Bash@3
# inputs:
# targetType: "inline"
# script: |
# set -ex
# supported_extensions=(py, sh, yaml)

# git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log
# files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++')

# for file in ${files}
# do
# if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
# echo "Checking license in 1 ${file}"
# if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
# echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
# fi
# else
# echo "Skipping ${file}"
# fi
# done

# - task: PublishPipelineArtifact@1
# inputs:
# targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
# artifact: copyright
# publishLocation: "pipeline"

# - task: Bash@3
# inputs:
# targetType: "inline"
# script: |
# ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
# if [ $? -eq 0 ] ; then
# echo "------------------Check <copyright_issue_summary.log> for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1
# fi
# exit 0
- stage: PyspellingCodeScan
displayName: Pyspelling Code Scan
dependsOn: []
jobs:
- job: Pyspelling
displayName: Pyspelling
steps:
- script: |
echo ${BUILD_SOURCESDIRECTORY}
sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
echo y | docker system prune
displayName: "Clean workspace"
- checkout: self
displayName: "Checkout out Repo"
- script: |
if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
fi
docker images | grep -i ${IMAGE_NAME}
if [[ $? -ne 0 ]]; then
echo "NO Such Image ${IMAGE_NAME}"
exit 1
fi
displayName: "Build Devel Images"
- script: |
docker stop $(docker ps -aq)
docker rm -vf $(docker ps -aq) || true
displayName: "Clean Docker"
- task: Bash@3
inputs:
targetType: "inline"
script: |
docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh
displayName: "Pyspelling Check"

- task: PublishPipelineArtifact@1
condition: always()
inputs:
targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log
artifact: pyspelling
publishLocation: "pipeline"
displayName: "PublishPipelineArtifact"

- stage: CopyRight
displayName: CopyRight Code Scan
dependsOn: []
jobs:
- job: CopyRight
displayName: CopyRight
steps:
- script: |
echo ${BUILD_SOURCESDIRECTORY}
sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
displayName: "Clean workspace"
- checkout: self
displayName: "Checkout out Repo"
- task: Bash@3
inputs:
targetType: "inline"
script: |
set -xe
mkdir -p $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)
supported_extensions=(py, sh, yaml)
git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log
files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++')
for file in ${files}
do
if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
echo "Checking license in 1 ${file}"
if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
echo "------------------Check <copyright_issue_summary.log> for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1
fi
else
echo "Skipping ${file}"
fi
done
displayName: "CopyRight Check"

- task: PublishPipelineArtifact@1
condition: failed()
inputs:
targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
artifact: copyright
publishLocation: "pipeline"
displayName: "PublishPipelineArtifact"
13 changes: 7 additions & 6 deletions .azure-pipelines/scripts/codeScan/bandit/bandit.sh
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
# set -ex
#!/bin/bash
set -ex

mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog

bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog"
pip install bandit


python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log

exit_code=$?
if [ ${exit_code} -eq 0 ] ; then
sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml
sed -i "s|ER|QR|g" /neural-compressor/azure-pipelines.yml

# sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml
if [ ${exit_code} -ne 0 ] ; then
echo "Bandit exited with non-zero exit code."; exit 1
fi
exit 0
Expand Down
31 changes: 0 additions & 31 deletions .azure-pipelines/scripts/codeScan/copyright/copy_right.sh
View file

This file was deleted.

0 comments on commit c1101be

Please sign in to comment.