diff --git a/dataprep/connector/generator/generator.py b/dataprep/connector/generator/generator.py index 802f6b845..bf6d31031 100644 --- a/dataprep/connector/generator/generator.py +++ b/dataprep/connector/generator/generator.py @@ -130,7 +130,14 @@ def save(self, path: Union[str, Path]) -> None: def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef: requests = Request(req["url"]) - resp = requests.post(_data=req["params"], _headers=req["headers"]) + if req["method"] == "GET": + resp = requests.get(_headers=req["headers"]) + elif req["method"] == "POST": + resp = requests.post(_data=req["params"], _headers=req["headers"]) + elif req["method"] == "PUT": + resp = requests.put(_data=req["params"], _headers=req["headers"]) + else: + raise RuntimeError(f"Unknown method {req['method']}") if resp.status != 200: raise RuntimeError(f"Request to HTTP endpoint not successful: {resp.status}: {resp.reason}") diff --git a/dataprep/connector/utils.py b/dataprep/connector/utils.py index a48acc8c7..7b4381452 100644 --- a/dataprep/connector/utils.py +++ b/dataprep/connector/utils.py @@ -18,8 +18,10 @@ class Request: def __init__(self, _url: str) -> None: self.url: urllib.parse.ParseResult = urllib.parse.urlparse(_url) + + qs = dict(urllib.parse.parse_qsl(self.url.query)) + self.url = self.url._replace(query=urllib.parse.urlencode(qs)) self.hostname: str = self.url.hostname or "" - self.path: str = self.url.path or "" self.headers: Dict[str, Any] = dict({"user-agent": "dataprep"}) def get(self, _headers: Optional[Dict[str, Any]] = None) -> http.client.HTTPResponse: @@ -34,8 +36,13 @@ def get(self, _headers: Optional[Dict[str, Any]] = None) -> http.client.HTTPResp self.headers.update(_headers) conn = http.client.HTTPSConnection(self.hostname) + path = self.url.path + if path == "": + path = "/" + if self.url.query: + path += "?" + self.url.query + conn.request(method="GET", url=path, headers=self.headers) - conn.request(method="GET", url=self.path, headers=self.headers) response = conn.getresponse() return response @@ -54,15 +61,22 @@ def post( if _headers: self.headers.update(_headers) conn = http.client.HTTPSConnection(self.hostname) + + path = self.url.path + if path == "": + path = "/" + if self.url.query: + path += "?" + self.url.query + if _data is not None: conn.request( method="POST", - url=self.path, + url=path, headers=self.headers, body=urllib.parse.urlencode(_data), ) else: - conn.request(method="POST", url=self.path, headers=self.headers) + conn.request(method="POST", url=path, headers=self.headers) response = conn.getresponse() return response @@ -81,15 +95,22 @@ def put( if _headers: self.headers.update(_headers) conn = http.client.HTTPSConnection(self.hostname) + + path = self.url.path + if path == "": + path = "/" + if self.url.query: + path += "?" + self.url.query + if _data is not None: conn.request( method="PUT", - url=self.path, + url=path, headers=self.headers, body=urllib.parse.urlencode(_data), ) else: - conn.request(method="PUT", url=self.path, headers=self.headers) + conn.request(method="PUT", url=path, headers=self.headers) response = conn.getresponse() return response