#27959: Simplifying even more OpenAI service classes and providing i…

…ntegration test for AIViewTool class

bom/application/pom.xml

@@ -1514,6 +1514,12 @@
                 <version>2.6</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.wiremock</groupId>
+                <artifactId>wiremock</artifactId>
+                <version>3.5.3</version>
+            </dependency>
+
             <!-- Graalvm Js Engine -->
             <dependency>
                 <groupId>org.graalvm.sdk</groupId>

dotCMS/src/main/java/com/dotcms/ai/service/OpenAIChatServiceImpl.java

@@ -33,7 +33,7 @@ public JSONObject sendRawRequest(final JSONObject prompt) {
 
         prompt.remove("prompt");
 
-        return new JSONObject(doRequest(config.getApiUrl(), "POST", config.getApiKey(), prompt));
+        return new JSONObject(doRequest(config.getApiUrl(), config.getApiKey(), prompt));
     }
 
     @Override
@@ -44,10 +44,8 @@ public JSONObject sendTextPrompt(final String textPrompt) {
     }
 
     @VisibleForTesting
-    String doRequest(final String urlIn,
-                     final String method,
-                     final String openAiAPIKey,
-                     final JSONObject json) {
-        return OpenAIRequest.doRequest(urlIn, method, openAiAPIKey, json);
+    String doRequest(final String urlIn, final String openAiAPIKey, final JSONObject json) {
+        return OpenAIRequest.doRequest(urlIn, "POST", openAiAPIKey, json);
     }
+
 }

dotCMS/src/main/java/com/dotcms/ai/service/OpenAIImageServiceImpl.java

@@ -70,7 +70,7 @@ public JSONObject sendRequest(final JSONObject jsonObject) {
 
         String responseString = "";
         try {
-            responseString = doRequest(config.getApiImageUrl(), "POST", config.getApiKey(), jsonObject);
+            responseString = doRequest(config.getApiImageUrl(), config.getApiKey(), jsonObject);
 
             JSONObject returnObject = new JSONObject(responseString);
             if (returnObject.containsKey("error")) {
@@ -89,6 +89,7 @@ public JSONObject sendRequest(final JSONObject jsonObject) {
         }
     }
 
+
     @Override
     public JSONObject sendRawRequest(final String prompt) {
         return sendRequest(new JSONObject(prompt));
@@ -112,7 +113,7 @@ public JSONObject sendTextPrompt(final String textPrompt) {
     private JSONObject createTempFile(final JSONObject imageResponse) {
         final String url = imageResponse.optString("url");
         if (UtilMethods.isEmpty(() -> url)) {
-            Logger.warn(this.getClass(), "imageResponse does not include URL:" + imageResponse.toString());
+            Logger.warn(this.getClass(), "imageResponse does not include URL:" + imageResponse);
             throw new DotRuntimeException("Image Response does not include URL:" + imageResponse);
         }
 
@@ -122,6 +123,7 @@ private JSONObject createTempFile(final JSONObject imageResponse) {
 
             final DotTempFile file = tempFileApi.createTempFileFromUrl(fileName, getRequest(), new URL(url), 20);
             imageResponse.put("response", file.id);
+            imageResponse.put("tempFile", file.file.getAbsolutePath());
 
             return imageResponse;
         } catch (Exception e) {
@@ -162,7 +164,7 @@ HttpServletRequest getRequest() {
         return requestProxy;
     }
 
-    private String generateFileName(final String originalPrompt){
+    private String generateFileName(final String originalPrompt) {
         final SimpleDateFormat dateToString = new SimpleDateFormat("yyyyMMdd_hhmmss");
         try {
             String newFileName = originalPrompt.toLowerCase();
@@ -183,11 +185,8 @@ private String generateFileName(final String originalPrompt){
     }
 
     @VisibleForTesting
-    String doRequest(final String urlIn,
-                     final String method,
-                     final String openAiAPIKey,
-                     final JSONObject json) {
-        return OpenAIRequest.doRequest(urlIn, method, openAiAPIKey, json);
+    String doRequest(final String urlIn, final String openAiAPIKey, final JSONObject json) {
+        return OpenAIRequest.doRequest(urlIn, "POST", openAiAPIKey, json);
     }
 
     @VisibleForTesting
@@ -200,7 +199,7 @@ AIImageRequestDTO.Builder getDtoBuilder() {
         return new AIImageRequestDTO.Builder();
     }
 
-    public static void setStopWordsUtil(StopWordsUtil stopWordsUtil) {
+    public static void setStopWordsUtil(final StopWordsUtil stopWordsUtil) {
         OpenAIImageServiceImpl.stopWordsUtil = stopWordsUtil;
     }
 

dotCMS/src/main/java/com/dotcms/ai/util/OpenAIRequest.java

@@ -7,119 +7,170 @@
 import com.dotmarketing.util.Logger;
 import com.dotmarketing.util.json.JSONObject;
 import io.vavr.control.Try;
+import org.apache.http.HttpHeaders;
 import org.apache.http.client.methods.*;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 
+import javax.ws.rs.core.MediaType;
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.OutputStream;
 import java.util.concurrent.ConcurrentHashMap;
 
+/**
+ * The OpenAIRequest class is a utility class that handles HTTP requests to the OpenAI API.
+ * It provides methods for sending GET, POST, PUT, DELETE, and PATCH requests.
+ * This class also manages rate limiting for the OpenAI API by keeping track of the last time a request was made.
+ *
+ * This class is implemented as a singleton, meaning that only one instance of the class is created throughout the execution of the program.
+ */
 public class OpenAIRequest {
 
-    static final ConcurrentHashMap<OpenAIModel,Long> lastRestCall = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<OpenAIModel,Long> lastRestCall = new ConcurrentHashMap<>();
 
+    private OpenAIRequest() {}
 
-
-
-    private OpenAIRequest() {
-    }
-
-    public static String doRequest(String url, String method, String openAiAPIKey, JSONObject json)  {
-
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
+    /**
+     * Sends a request to the specified URL with the specified method, OpenAI API key, and JSON payload.
+     * The response from the request is returned as a string.
+     *
+     * @param url the URL to send the request to
+     * @param method the HTTP method to use for the request
+     * @param openAiAPIKey the OpenAI API key to use for the request
+     * @param json the JSON payload to send with the request
+     * @return the response from the request as a string
+     */
+    public static String doRequest(final String url,
+                                   final String method,
+                                   final String openAiAPIKey,
+                                   final JSONObject json)  {
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
         doRequest(url, method, openAiAPIKey, json, out);
-        return out.toString();
-
 
+        return out.toString();
     }
 
-
-    private static HttpUriRequest resolveMethod(String method, String urlIn) {
-        if ("post" .equalsIgnoreCase(method)) {
-            return new HttpPost(urlIn);
-        }
-        if ("put" .equalsIgnoreCase(method)) {
-            return new HttpPut(urlIn);
-        }
-        if ("delete" .equalsIgnoreCase(method)) {
-            return new HttpDelete(urlIn);
-        }
-        if ("patch" .equalsIgnoreCase(method)) {
-            return new HttpPatch(urlIn);
-        }
-
-        return  new HttpGet(urlIn);
-
-    }
-    public static void doPost(String urlIn,  String openAiAPIKey, JSONObject json, OutputStream out) {
+    /**
+     * Sends a POST request to the specified URL with the specified OpenAI API key and JSON payload.
+     * The response from the request is written to the provided OutputStream.
+     *
+     * @param urlIn the URL to send the request to
+     * @param openAiAPIKey the OpenAI API key to use for the request
+     * @param json the JSON payload to send with the request
+     * @param out the OutputStream to write the response to
+     */
+    public static void doPost(final String urlIn,
+                              final String openAiAPIKey,
+                              final JSONObject json,
+                              final OutputStream out) {
        doRequest(urlIn,"post",openAiAPIKey,json,out);
     }
 
-    public static void doGet(String urlIn,  String openAiAPIKey, JSONObject json, OutputStream out) {
+    /**
+     * Sends a GET request to the specified URL with the specified OpenAI API key and JSON payload.
+     * The response from the request is written to the provided OutputStream.
+     *
+     * @param urlIn the URL to send the request to
+     * @param openAiAPIKey the OpenAI API key to use for the request
+     * @param json the JSON payload to send with the request
+     * @param out the OutputStream to write the response to
+     */
+    public static void doGet(final String urlIn,
+                             final String openAiAPIKey,
+                             final JSONObject json,
+                             final OutputStream out) {
         doRequest(urlIn,"get",openAiAPIKey,json,out);
     }
 
-
     /**
-     * this allows for a streaming response.  It also attempts to rate limit requests based on OpenAI limits
-     * @param urlIn
-     * @param method
-     * @param openAiAPIKey
-     * @param json
-     * @param out
+     * Sends a request to the specified URL with the specified method, OpenAI API key, and JSON payload.
+     * The response from the request is written to the provided OutputStream.
+     * This method also manages rate limiting for the OpenAI API by keeping track of the last time a request was made.
+     *
+     * @param urlIn the URL to send the request to
+     * @param method the HTTP method to use for the request
+     * @param openAiAPIKey the OpenAI API key to use for the request
+     * @param json the JSON payload to send with the request
+     * @param out the OutputStream to write the response to
      */
-    public static void doRequest(String urlIn, String method, String openAiAPIKey, JSONObject json, OutputStream out) {
-        if(ConfigService.INSTANCE.config().getConfigBoolean(AppKeys.DEBUG_LOGGING)) {
-            Logger.info(OpenAIRequest.class, "posting:" + json);
+    public static void doRequest(final String urlIn,
+                                 final String method,
+                                 final String openAiAPIKey,
+                                 final JSONObject json,
+                                 final OutputStream out) {
+
+        if (ConfigService.INSTANCE.config().getConfigBoolean(AppKeys.DEBUG_LOGGING)) {
+            Logger.debug(OpenAIRequest.class, "posting:" + json);
         }
-        final OpenAIModel model = OpenAIModel.resolveModel(json.optString("model"));
-
 
-        long sleep = lastRestCall.computeIfAbsent(model, m -> 0L) + model.minIntervalBetweenCalls() - System.currentTimeMillis();
+        final OpenAIModel model = OpenAIModel.resolveModel(json.optString("model"));
+        final long sleep = lastRestCall.computeIfAbsent(model, m -> 0L)
+                + model.minIntervalBetweenCalls()
+                - System.currentTimeMillis();
         if (sleep > 0) {
-            Logger.info(OpenAIRequest.class, "Rate limit:" + model.apiPerMinute + "/minute, or 1 every " + (60000 / model.apiPerMinute) + "ms. Sleeping:" + sleep);
+            Logger.info(
+                    OpenAIRequest.class,
+                    "Rate limit:"
+                            + model.apiPerMinute
+                            + "/minute, or 1 every "
+                            + (60000 / model.apiPerMinute)
+                            + "ms. Sleeping:"
+                            + sleep);
             Try.run(() -> Thread.sleep(sleep));
         }
+
         lastRestCall.put(model, System.currentTimeMillis());
 
         try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+            final StringEntity jsonEntity = new StringEntity(json.toString(), ContentType.APPLICATION_JSON);
+            final HttpUriRequest httpRequest = resolveMethod(method, urlIn);
+            httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON);
+            httpRequest.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + openAiAPIKey);
 
-
-            StringEntity jsonEntity = new StringEntity(json.toString(), ContentType.APPLICATION_JSON);
-            HttpUriRequest httpRequest = resolveMethod(method, urlIn);
-            httpRequest.setHeader("Content-Type", "application/json");
-            httpRequest.setHeader("Authorization", "Bearer " + openAiAPIKey);
-            if (null !=json && !json.getAsMap().isEmpty()) {
+            if (!json.getAsMap().isEmpty()) {
                 Try.run(() -> ((HttpEntityEnclosingRequestBase) httpRequest).setEntity(jsonEntity));
             }
+
             try (CloseableHttpResponse response = httpClient.execute(httpRequest)) {
-                BufferedInputStream in = new BufferedInputStream(response.getEntity().getContent());
-                byte[] buffer = new byte[1024];
+                final BufferedInputStream in = new BufferedInputStream(response.getEntity().getContent());
+                final byte[] buffer = new byte[1024];
                 int len;
                 while ((len = in.read(buffer)) != -1) {
                     out.write(buffer, 0, len);
                     out.flush();
                 }
-
             }
-
         } catch (Exception e) {
-            if(ConfigService.INSTANCE.config().getConfigBoolean(AppKeys.DEBUG_LOGGING)){
-                Logger.warn(OpenAIRequest.class, "INVALID REQUEST: " + e.getMessage(),e);
-                Logger.warn(OpenAIRequest.class, " -  " + method + " : " +json.toString());
-            }else{
+            if (ConfigService.INSTANCE.config().getConfigBoolean(AppKeys.DEBUG_LOGGING)){
+                Logger.warn(OpenAIRequest.class, "INVALID REQUEST: " + e.getMessage(), e);
+            } else {
                 Logger.warn(OpenAIRequest.class, "INVALID REQUEST: " + e.getMessage());
-                Logger.warn(OpenAIRequest.class, " -  " + method + " : " +json.toString());
             }
 
+            Logger.warn(OpenAIRequest.class, " -  " + method + " : " +json);
+
             throw new DotRuntimeException(e);
         }
-
     }
 
+    private static HttpUriRequest resolveMethod(final String method, final String urlIn) {
+        if ("post".equalsIgnoreCase(method)) {
+            return new HttpPost(urlIn);
+        }
+        if ("put".equalsIgnoreCase(method)) {
+            return new HttpPut(urlIn);
+        }
+        if ("delete".equalsIgnoreCase(method)) {
+            return new HttpDelete(urlIn);
+        }
+        if ("patch".equalsIgnoreCase(method)) {
+            return new HttpPatch(urlIn);
+        }
+
+        return  new HttpGet(urlIn);
+    }
 
 }

dotCMS/src/main/java/com/dotcms/ai/util/StopWordsUtil.java

@@ -9,6 +9,16 @@
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+/**
+ * This utility class is used for handling stop words in a given text.
+ * Stop words are words which are filtered out during the processing of text.
+ * When building the vocabulary of a text data, it may be a good idea to consider these words as noise.
+ *
+ * The class provides a method to remove stop words from a given string.
+ * It uses a predefined set of stop words, which are common words that do not contain important meaning and are usually removed from texts.
+ *
+ * This class is implemented as a singleton, meaning that only one instance of the class is created throughout the execution of the program.
+ */
 public class StopWordsUtil {
 
     // twitter stopwords - stopwords_twitter.txt
@@ -61,12 +71,26 @@ public class StopWordsUtil {
 
     private static final Lazy<StopWordsUtil> STOP_WORDS_UTILS = Lazy.of(StopWordsUtil::new);
 
+    /**
+     * Returns the singleton instance of the StopWordsUtil class.
+     * The instance is created only once and reused for all subsequent calls.
+     *
+     * @return the singleton instance of the StopWordsUtil class
+     */
     public static StopWordsUtil get(){
         return STOP_WORDS_UTILS.get();
     }
 
     private StopWordsUtil() {}
 
+    /**
+     * Removes all stop words from the given string.
+     * The input string is split into words, and any word that is a stop word is removed.
+     * The remaining words are joined back together with spaces in between.
+     *
+     * @param incoming the string from which to remove stop words
+     * @return a new string with all stop words removed
+     */
     public String removeStopWords(final String incoming) {
         return Optional
                 .ofNullable(incoming)