Multi threads in multiple processes

[lionsheep0724]

Hello team!
I want to deploy a server with bentoml(0.13.1), which can asynchronously handle multiple requests, spawn processes when requests are arrived to server and each process has multiple threads to work.
I tried to call ThreadPoolExecutor inside ProcessPoolExecutor, with --workers = N option but the error "daemonic processes are not allowed to have children" occurred. I guess it crashed with aiohttp in bentoml.
How do I handle this problem?? please refer to below.

[bojiang]

Hi @lionsheep0724.

tried to call ThreadPoolExecutor inside ProcessPoolExecutor

Could you please share the code for this part?

[lionsheep0724]

Hi @bojiang .
Let me share related codesfor the part you mentioned.

def get_cropped_image_list(self, input: List):
        task_results = []
        with ThreadPoolExecutor(max_workers=16) as exe:
            response = exe.map(self.read_crop_save_image, input)

        for r in response:
            task_results.extend(r)

        return task_results
def preprocess(self, legacy_prediction_data_frame: pd.DataFrame):
     
        
        cropped_image_list = self.get_cropped_image_list(input)

        return cropped_image_list

def inference(self, cropped_image_list: List):
        pytorch_dataset = MyDataset(arr=cropped_image_list)
        pytorch_data_loader = DataLoader(pytorch_dataset)
        outputs = []
        self.artifacts.model.eval()
        
        with torch.no_grad():
            for cropped_image in pytorch_data_loader:
           
                output = self.artifacts.model(cropped_image .to(configs.DEVICE))
                outputs.append(output.cpu().numpy())
    
        return outputs

@api(input=DataframeInput(),batch=True,mb_max_latency=60000,mb_max_batch_size=5000)
    def predict(self, input_dataframes: pd.DataFrame):
        return_tables= []
        input_dataframes.set_index('index',inplace=True)
        input_data_frame_list = self.split_dataframe(input_dataframes)
     
        for input_dataframe in input_data_frame_list :
     
            cropped_image_list = self.preprocess(input_dataframe )
            outputs = self.inference(cropped_image_list )
            inference_end = datetime.now()
            
            return_table= self.postprocess(outputs)
     
            return_tables.append(return_table)

        return pd.concat(return_tables).reset_index(drop=True)