From f37c30efe7588a071d9a4f0737c170685778d8fc Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 16:52:53 +0100
Subject: [PATCH 1/5] =?UTF-8?q?feat:=20script=20R=20pour=20l'export=20de?=
 =?UTF-8?q?=20donn=C3=A9es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 bin/export.r                                  | 144 ++++++++++++++++++
 .../sido/gwt/server/ws/QueryServiceImpl.java  |  14 +-
 2 files changed, 153 insertions(+), 5 deletions(-)
 create mode 100755 bin/export.r

diff --git a/bin/export.r b/bin/export.r
new file mode 100755
index 00000000..fd018772
--- /dev/null
+++ b/bin/export.r
@@ -0,0 +1,144 @@
+#!/usr/bin/Rscript --vanilla --no-save --no-restore --quiet --encoding=UTF-8
+# Call SIDO web service to get data
+#
+## Uses
+## - RCurl: https://cran.r-project.org/web/packages/RCurl/index.html
+## - keyring: https://cran.r-project.org/web/packages/keyring/index.html
+## - logger: https://cran.r-project.org/web/packages/logger/index.html
+
+sido_url <- "https://sido.pheno.fr/ws"
+observatory_schema <- "foret"
+
+#' Load or install a package.
+#'
+#' @param packagename string package name
+require_install_package <- function(packagename) {
+  if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+    cat("The library", packagename, "is not installed. Trying to install it.\n")
+    install.packages(packagename, repos = "http://cran.at.r-project.org/")
+    if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+      cat("Installing the library", packagename, "failed.\n")
+      q()
+    }
+  }
+}
+
+require_install_package("RCurl")
+require_install_package("jsonlite")
+require_install_package("keyring")
+require_install_package("logger")
+
+#' Get key value from keyring.
+#'
+#' @param key_name string key name
+#' @param prompt string prompt message
+#' @return string keyring value
+get_keyring_value <- function(key_name, prompt) {
+  logger::log_info("Getting keyring value for {key_name}...")
+  if (!keyring::has_keyring_support()) {
+    logger::log_error("No keyring support!")
+    logger::log_info("Set value for the variable sido_{key_name} in the code.")
+    q()
+  }
+  keyring_service <- "sido-export"
+
+  kr <- keyring::default_backend(keyring = keyring_service)
+  if (!kr$is_available()) {
+    logger::log_error("Keyring {keyring_service} not available!")
+    logger::log_info("Set value for the variable sido_{key_name} in the code.")
+    q()
+  }
+
+  key_list <- kr$list()
+  match_keys <- subset(key_list,
+                       service == keyring_service & username == key_name)
+  if (dim(match_keys)[1] == 0) {
+    logger::log_info("No value found for {keyring_service} and {key_name}.")
+    kr$set(service = keyring_service, username = key_name, prompt = prompt)
+  }
+  value <- kr$get(service = keyring_service, username = key_name)
+  value
+}
+
+#' Get access token using client credentials.
+#'
+#' @return access token
+get_access_token <- function(sido_client_id, sido_client_secret) {
+  logger::log_info("Getting access token...")
+  token_url <- paste0(sido_url, "/oauth/token")
+  # Perform the POST request
+  post_data <- paste0(
+                      "client_id=", sido_client_id,
+                      "&client_secret=", sido_client_secret,
+                      "&grant_type=client_credentials")
+  headers <- c("Content-Type" = "application/x-www-form-urlencoded")
+  opts <- list(postfields = post_data, httpheader = headers)
+  logger::log_info("Calling {token_url}...")
+  post_response <- RCurl::postForm(token_url, .opts = opts)
+  logger::log_info("Parsing JSON...")
+  token <- jsonlite::fromJSON(post_response)
+  token$access_token
+}
+
+#' Get queries for a given schema.
+#'
+#' @param schema string schema name
+#' @param access_token string access token
+#' @return queries list of query names
+get_queries <- function(schema, access_token) {
+  logger::log_info("Getting queries...")
+  queries_url <- paste0(sido_url, "/data/queries/", schema)
+  headers <- c("Authorization" = paste0("Bearer ", access_token))
+  queries_response <- RCurl::getURL(queries_url,
+                                    .opts = list(httpheader = headers))
+  queries <- jsonlite::fromJSON(queries_response)
+  queries$data[, "name"]
+}
+
+#' Get data for a given schema and query.
+#'
+#' @param schema string schema name
+#' @param query string query name
+#' @param access_token string access token
+#' @return data data frame
+get_data <- function(schema, query, access_token) {
+  logger::log_info("- Getting data for {schema}/{query}...")
+  data_url <- paste0(sido_url, "/data/", schema, "/", query)
+  headers <- c("Authorization" = paste0("Bearer ", access_token))
+  data_response <- RCurl::getURL(data_url,
+                                 .opts = list(httpheader = headers),
+                                 .encoding = "UTF-8")
+  data <- jsonlite::fromJSON(data_response)
+  if (data$status != 200) {
+    logger::log_error("- Error ", data$status, ": ", data$message, "\n")
+  }
+  df <- data.frame(data$data$values)
+  names(df) <- data$data$properties$title
+  df
+}
+
+sido_client_id <- get_keyring_value("client_id",
+                                    "Your client_id for SIDO")
+sido_client_secret <- get_keyring_value("client_secret",
+                                        "Your client_secret for SIDO")
+
+token <- get_access_token(sido_client_id, sido_client_secret)
+queries <- get_queries(observatory_schema, token)
+logger::log_info("- nb of queries: {0}", length(queries))
+for (query in queries) {
+  logger::log_info("- query: {query}")
+  if (query == "data") {
+    logger::log_info("  - skipping data query")
+  } else {
+    data <- get_data(observatory_schema, query, token)
+    filename <- paste0(observatory_schema, "-", query, ".csv")
+    logger::log_info("  - file: {filename}...")
+    write.table(data,
+                file = filename,
+                col.names = TRUE,
+                row.names = FALSE,
+                sep = ",",
+                quote = TRUE)
+    logger::log_info("    done")
+  }
+}
diff --git a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
index 73619af3..8a5a10b8 100644
--- a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
+++ b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
@@ -112,12 +112,16 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public final WsQuery getQuery(@NonNull final Datasource obs,
-            final String queryName) {
-        if (getQueries(obs) != null) {
-            return getQueries(obs).get(0);
-        } else {
-            return null;
+            @NonNull final String queryName) {
+        final var queries = getQueries(obs);
+        if (queries != null) {
+            for (final var query : queries) {
+                if (queryName.equals(query.getName())) {
+                    return query;
+                }
+            }
         }
+        return null;
     }
 
     @Override
-- 
GitLab


From c8c93907038a5fcc24e3f140dfea56b26983f673 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 17:19:17 +0100
Subject: [PATCH 2/5] Revue de code

---
 bin/export.r                                             | 4 ++--
 .../soeretempo/sido/gwt/server/ws/QueryServiceImpl.java  | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/bin/export.r b/bin/export.r
index fd018772..a8c33dd2 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -14,10 +14,10 @@ observatory_schema <- "foret"
 #' @param packagename string package name
 require_install_package <- function(packagename) {
   if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
-    cat("The library", packagename, "is not installed. Trying to install it.\n")
+    logger::log_info("Installing the library {packagename}...")
     install.packages(packagename, repos = "http://cran.at.r-project.org/")
     if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
-      cat("Installing the library", packagename, "failed.\n")
+      logger::log_error("Installing the library {packagename} failed.")
       q()
     }
   }
diff --git a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
index 8a5a10b8..b18d308c 100644
--- a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
+++ b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
@@ -115,11 +115,10 @@ public class QueryServiceImpl implements QueryService {
             @NonNull final String queryName) {
         final var queries = getQueries(obs);
         if (queries != null) {
-            for (final var query : queries) {
-                if (queryName.equals(query.getName())) {
-                    return query;
-                }
-            }
+            return queries.stream()
+                    .filter(query -> queryName.equals(query.getName()))
+                    .findFirst()
+                    .orElse(null);
         }
         return null;
     }
-- 
GitLab


From e3de4a49a442c6f2e76f27aada799f833dfa0c64 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 17:22:14 +0100
Subject: [PATCH 3/5] Revue de code

---
 bin/export.r | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/export.r b/bin/export.r
index a8c33dd2..ecfcddeb 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -110,7 +110,7 @@ get_data <- function(schema, query, access_token) {
                                  .encoding = "UTF-8")
   data <- jsonlite::fromJSON(data_response)
   if (data$status != 200) {
-    logger::log_error("- Error ", data$status, ": ", data$message, "\n")
+    logger::log_error(paste0("  Error ", data$status, ": ", data$message))
   }
   df <- data.frame(data$data$values)
   names(df) <- data$data$properties$title
-- 
GitLab


From a94b4ada9ec9841c0c6b57e0117515d77c7bd9d5 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Thu, 20 Feb 2025 16:43:37 +0100
Subject: [PATCH 4/5] refactor: prise en compte de la revue de code R de
 Patrice Lecharpentier

---
 bin/export.r | 67 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 42 insertions(+), 25 deletions(-)

diff --git a/bin/export.r b/bin/export.r
index ecfcddeb..8a225f1e 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -6,21 +6,23 @@
 ## - keyring: https://cran.r-project.org/web/packages/keyring/index.html
 ## - logger: https://cran.r-project.org/web/packages/logger/index.html
 
-sido_url <- "https://sido.pheno.fr/ws"
+sido_ws_url <- "https://sido.pheno.fr/ws"
 observatory_schema <- "foret"
+keyring_service <- "sido-export"
 
 #' Load or install a package.
 #'
 #' @param packagename string package name
 require_install_package <- function(packagename) {
-  if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+  if (!library(packagename, character.only=TRUE, logical.return=TRUE)) {
     logger::log_info("Installing the library {packagename}...")
     install.packages(packagename, repos = "http://cran.at.r-project.org/")
-    if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+    if (!library(packagename, character.only=TRUE, logical.return=TRUE)) {
       logger::log_error("Installing the library {packagename} failed.")
       q()
     }
   }
+  require(packagename, quietly = TRUE, character.only = TRUE)
 }
 
 require_install_package("RCurl")
@@ -28,26 +30,33 @@ require_install_package("jsonlite")
 require_install_package("keyring")
 require_install_package("logger")
 
+#' Get keyring
+#'
+#' @return keyring for the script
+get_keyring <- function(keyring_service) {
+  logger::log_info("Using backend_file")
+  #kr <- keyring::backend_file$new(keyring = keyring_service)
+  kr <- keyring::default_backend(keyring = keyring_service)
+  if (kr$keyring_is_locked(keyring=keyring_service)) {
+    kr$keyring_unlock(keyring = keyring_service)
+  }
+  kr
+}
+
 #' Get key value from keyring.
 #'
+#' @param kr keyring for the script
 #' @param key_name string key name
 #' @param prompt string prompt message
 #' @return string keyring value
-get_keyring_value <- function(key_name, prompt) {
+get_keyring_value <- function(kr, key_name, prompt) {
   logger::log_info("Getting keyring value for {key_name}...")
   if (!keyring::has_keyring_support()) {
     logger::log_error("No keyring support!")
     logger::log_info("Set value for the variable sido_{key_name} in the code.")
     q()
   }
-  keyring_service <- "sido-export"
-
-  kr <- keyring::default_backend(keyring = keyring_service)
-  if (!kr$is_available()) {
-    logger::log_error("Keyring {keyring_service} not available!")
-    logger::log_info("Set value for the variable sido_{key_name} in the code.")
-    q()
-  }
+  keyring_service <- kr$keyring_default()
 
   key_list <- kr$list()
   match_keys <- subset(key_list,
@@ -56,14 +65,16 @@ get_keyring_value <- function(key_name, prompt) {
     logger::log_info("No value found for {keyring_service} and {key_name}.")
     kr$set(service = keyring_service, username = key_name, prompt = prompt)
   }
-  value <- kr$get(service = keyring_service, username = key_name)
-  value
+  kr$get(service = keyring_service, username = key_name)
 }
 
 #' Get access token using client credentials.
 #'
+#' @param sido_url string root URL of SIDO WS
+#' @param sido_client_id string OAuth2 client ID for SIDO
+#' @param sido_client_secret string OAuth2 client secret for SIDO
 #' @return access token
-get_access_token <- function(sido_client_id, sido_client_secret) {
+get_access_token <- function(sido_url, sido_client_id, sido_client_secret) {
   logger::log_info("Getting access token...")
   token_url <- paste0(sido_url, "/oauth/token")
   # Perform the POST request
@@ -82,10 +93,11 @@ get_access_token <- function(sido_client_id, sido_client_secret) {
 
 #' Get queries for a given schema.
 #'
+#' @param sido_url string root URL of SIDO WS
 #' @param schema string schema name
 #' @param access_token string access token
 #' @return queries list of query names
-get_queries <- function(schema, access_token) {
+get_queries <- function(sido_url, schema, access_token) {
   logger::log_info("Getting queries...")
   queries_url <- paste0(sido_url, "/data/queries/", schema)
   headers <- c("Authorization" = paste0("Bearer ", access_token))
@@ -97,11 +109,12 @@ get_queries <- function(schema, access_token) {
 
 #' Get data for a given schema and query.
 #'
+#' @param sido_url string root URL of SIDO WS
 #' @param schema string schema name
 #' @param query string query name
 #' @param access_token string access token
 #' @return data data frame
-get_data <- function(schema, query, access_token) {
+get_data <- function(sido_url, schema, query, access_token) {
   logger::log_info("- Getting data for {schema}/{query}...")
   data_url <- paste0(sido_url, "/data/", schema, "/", query)
   headers <- c("Authorization" = paste0("Bearer ", access_token))
@@ -117,20 +130,24 @@ get_data <- function(schema, query, access_token) {
   df
 }
 
-sido_client_id <- get_keyring_value("client_id",
-                                    "Your client_id for SIDO")
-sido_client_secret <- get_keyring_value("client_secret",
-                                        "Your client_secret for SIDO")
+# Run
+
+kr <- get_keyring(keyring_service)
+sido_client_id <- get_keyring_value(kr, "client_id",
+                                    "Type your client_id for SIDO")
+sido_client_secret <- get_keyring_value(kr, "client_secret",
+                                        "Type your client_secret for SIDO")
 
-token <- get_access_token(sido_client_id, sido_client_secret)
-queries <- get_queries(observatory_schema, token)
+token <- get_access_token(sido_ws_url, sido_client_id, sido_client_secret)
+queries <- get_queries(sido_ws_url, observatory_schema, token)
 logger::log_info("- nb of queries: {0}", length(queries))
-for (query in queries) {
+for (i in seq_along(queries)) {
+  query = queries[i]
   logger::log_info("- query: {query}")
   if (query == "data") {
     logger::log_info("  - skipping data query")
   } else {
-    data <- get_data(observatory_schema, query, token)
+    data <- get_data(sido_ws_url, observatory_schema, query, token)
     filename <- paste0(observatory_schema, "-", query, ".csv")
     logger::log_info("  - file: {filename}...")
     write.table(data,
-- 
GitLab


From ef1910909fe50b79b454f63017e3aab35d548664 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Fri, 21 Feb 2025 10:21:04 +0100
Subject: [PATCH 5/5] Suppression de commentaire

---
 bin/export.r | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/export.r b/bin/export.r
index 8a225f1e..1fdc4f3a 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -35,7 +35,6 @@ require_install_package("logger")
 #' @return keyring for the script
 get_keyring <- function(keyring_service) {
   logger::log_info("Using backend_file")
-  #kr <- keyring::backend_file$new(keyring = keyring_service)
   kr <- keyring::default_backend(keyring = keyring_service)
   if (kr$keyring_is_locked(keyring=keyring_service)) {
     kr$keyring_unlock(keyring = keyring_service)
-- 
GitLab