Update to function as out-of-the-box test server

README.md

@@ -6,7 +6,7 @@ Overview of all OpenML components including a docker-compose to run OpenML servi
 ![OpenML Component overview](https://raw.githubusercontent.com/openml/services/main/documentation/OpenML-overview.png)
 
 ## Prerequisites
-- Linux/MacOS with Intell processor (because of our old ES version, this project currently does not support `arm` architectures)
+- Linux/MacOS (For Mac with `arm` architectures, enable Rosetta for emulation. QEMU and Docker VMM do not work with the elastic search image)
 - [Docker](https://docs.docker.com/get-docker/) 
 - [Docker Compose](https://docs.docker.com/compose/install/) version 2.21.0 or higher
 
@@ -122,11 +122,17 @@ Make sure the code in your frontend directory is build, by running (from your lo
 ### Python
 
 You can run the openml-python code on your own local server now!
+Normally, you would be able to run the following command to start a Python console:
 
 ```bash
-docker run --rm -it -v ./config/python/config:/root/.config/openml/config:ro --network openml-services openml/openml-python
+docker run --rm -it -v ./config/python/config:/root/.config/openml/config:ro --network openml-services --add-host localhost:172.28.0.2 openml/openml-python
 ```
 
+But the current image is a bit out of date so instead start the container with:
+
+```bash
+docker run --rm -it -v ./config/python/config:/root/.config/openml/config:ro --network openml-services --add-host localhost:172.28.0.2 --entrypoint=/bin/bash openml/openml-python -c "/openml/venv/bin/python -m pip install -U pyarrow pandas  && cd /openml && /openml/venv/bin/python"
+```
 
 For an example of manual tests, you can run:
 ```python

config/arff-to-pq-converter/Dockerfile

@@ -8,4 +8,4 @@ RUN apt update && apt upgrade -y
 RUN apt -y install cron
 RUN chmod +x /etc/cron.d/openml
 
-RUN crontab -u unprivileged-user /etc/cron.d/openml
+RUN crontab -u unprivileged-user /etc/cron.d/openml

config/arff-to-pq-converter/config

@@ -1 +1 @@
-server=http://nginx:80/api/v1/xml
+server=http://nginx:8000/api/v1/xml

config/database/update.sh

@@ -2,30 +2,7 @@
 # Change the filepath of openml.file 
 # from "https://www.openml.org/data/download/1666876/phpFsFYVN"
 # to "http://minio:9000/datasets/0000/0001/phpFsFYVN"
-mysql -hdatabase -uroot -pok -e 'UPDATE openml.file SET filepath = CONCAT("http://minio:9000/datasets/0000/", LPAD(id, 4, "0"), "/", SUBSTRING_INDEX(filepath, "/", -1)) WHERE extension="arff";'
+mysql -hdatabase -uroot -pok -e 'UPDATE openml.file SET filepath = CONCAT("http://localhost:8000/minio/datasets/0000/", LPAD(id, 4, "0"), "/", SUBSTRING_INDEX(filepath, "/", -1)) WHERE extension="arff";'
 
 # Update openml.expdb.dataset with the same url
 mysql -hdatabase -uroot -pok -e 'UPDATE openml_expdb.dataset DS, openml.file FL SET DS.url = FL.filepath WHERE DS.did = FL.id;'
-
-
-
-
-
-# Create the data_feature_description TABLE. TODO: can we make sure this table exists already?
-mysql -hdatabase -uroot -pok -Dopenml_expdb -e 'CREATE TABLE IF NOT EXISTS `data_feature_description` (
-  `did` int unsigned NOT NULL,
-  `index` int unsigned NOT NULL,
-  `uploader` mediumint unsigned NOT NULL,
-  `date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `description_type` enum("plain", "ontology") NOT NULL,
-  `value` varchar(256) NOT NULL,
-  KEY `did` (`did`,`index`),
-  CONSTRAINT `data_feature_description_ibfk_1` FOREIGN KEY (`did`, `index`) REFERENCES `data_feature` (`did`, `index`) ON DELETE CASCADE ON UPDATE CASCADE
-)'
-
-# SET dataset 1 to active (used in unittests java)
-mysql -hdatabase -uroot -pok -Dopenml_expdb -e 'INSERT IGNORE INTO dataset_status VALUES (1, "active", "2024-01-01 00:00:00", 1)'
-mysql -hdatabase -uroot -pok -Dopenml_expdb -e 'DELETE FROM dataset_status WHERE did = 2 AND status = "deactivated";'
-
-# Temporary fix in case the database missed the kaggle table. The PHP Rest API expects the table to be there, while indexing.
-mysql -hdatabase -uroot -pok -Dopenml_expdb -e 'CREATE TABLE IF NOT EXISTS `kaggle` (`dataset_id` int(11) DEFAULT NULL, `kaggle_link` varchar(500) DEFAULT NULL)'

config/evaluation-engine/.env

@@ -1,4 +1,4 @@
-CONFIG=api_key=AD000000000000000000000000000000;server=http://php-api:80/
+CONFIG=api_key=abc;server=http://php-api:80/
 JAVA=/usr/bin/java
 JAR=/usr/local/lib/evaluation-engine.jar
-LOG_DIR=/logs
+LOG_DIR=/logs

config/evaluation-engine/run-cron.sh

@@ -1,6 +1,16 @@
 #!/bin/sh
 
+# We need to remove the default 127.0.0.1 localhost map to
+# ensure the remap to the static nginx ip address is respected.
+# Updating /etc/hosts in place isn't always allowed ("Resource Busy"),
+# directly overwriting it instead seems to bypass that protection.
+cp /etc/hosts /etc/hosts.new
+sed -i '/^127.0.0.1.*localhost/d' /etc/hosts.new
+sed -i -E 's/^(::1\t)localhost (.*)$/\1\2/g' /etc/hosts.new
+cat /etc/hosts.new > /etc/hosts
+rm /etc/hosts.new
+
 printenv | grep -v HOME >> /etc/environment
 
 touch /cron.log
-/usr/sbin/crond -l 4 && tail -f /cron.log
+/usr/sbin/crond -l 4 && tail -f /cron.log

config/frontend/.env

@@ -9,7 +9,7 @@ DATABASE_URI="mysql+pymysql://root:ok@database:3306/openml"
 TESTING=False
 
 REACT_APP_URL_SITE_BACKEND=http://localhost:8000/
-REACT_APP_URL_API=http://localhost:8000/api/
+REACT_APP_URL_API=http://localhost:8000/
 REACT_APP_URL_ELASTICSEARCH=http://localhost:8000/es/
 REACT_APP_ELASTICSEARCH_VERSION_MAYOR=6
 REACT_APP_URL_MINIO=http://localhost:8000/data/

config/nginx/Dockerfile

@@ -3,6 +3,6 @@ FROM nginx:alpine
 WORKDIR /etc/nginx
 COPY ./nginx.conf ./conf.d/default.conf
 COPY ./shared.conf ./shared.conf
-EXPOSE 80
+EXPOSE 8000
 ENTRYPOINT [ "nginx" ]
-CMD [ "-g", "daemon off;" ]
+CMD [ "-g", "daemon off;" ]

config/nginx/nginx.conf

@@ -3,49 +3,49 @@
 
 
 server {
-  listen 80;
+  listen 8000;
   server_name localhost;
   resolver 127.0.0.11;
 
   location ~ ^/es(?:\/(.*))?$ {
      include /etc/nginx/shared.conf; 
      set $upstream_es http://elasticsearch:9200;
-     proxy_pass $upstream_es/$1$is_args;
+     proxy_pass $upstream_es/$1$is_args$args;
   }
 
   location ~ ^/api_splits/(.*)$ {
      include /etc/nginx/shared.conf; 
      set $upstream_api http://php-api:80/api_splits;
-     proxy_pass $upstream_api/$1$is_args;
+     proxy_pass $upstream_api/$1$is_args$args;
   }
 
   location ~ ^/api(?:\/(.*))$ {
      include /etc/nginx/shared.conf; 
      set $upstream_api http://php-api:80/api;
-     proxy_pass $upstream_api/$1$is_args;
+     proxy_pass $upstream_api/$1$is_args$args;
   }
 
   location ~ ^/croissant/dataset/(.*)$ {
      include /etc/nginx/shared.conf; 
      set $upstream_data http://croissants:8001;
-     proxy_pass $upstream_data/$1$is_args;
+     proxy_pass $upstream_data/$1$is_args$args;
   }
 
   location ~ ^/data(?:\/(.*))$ {
      include /etc/nginx/shared.conf; 
      set $upstream_api http://php-api:80/data;
-     proxy_pass $upstream_api/$1$is_args;
+     proxy_pass $upstream_api/$1$is_args$args;
   }
 
   location ~ ^/minio/(.*)$ {
      include /etc/nginx/shared.conf; 
      set $upstream_data http://minio:9000;
-     proxy_pass $upstream_data/$1$is_args;
+     proxy_pass $upstream_data/$1$is_args$args;
   }
 
   location ~ ^(?:\/(.*))?$ {
      include /etc/nginx/shared.conf; 
      set $upstream_f http://frontend:5000;
-     proxy_pass $upstream_f/$1$is_args;
+     proxy_pass $upstream_f/$1$is_args$args;
   }
-}
+}

config/php/.env

@@ -1,6 +1,6 @@
-API_KEY=AD000000000000000000000000000000	
-BASE_URL=http://php-api:80/
-MINIO_URL=http://minio:9000/
+API_KEY=abc
+BASE_URL=http://localhost:8000/
+MINIO_URL=http://localhost:8000/minio/
 DB_HOST_OPENML=database:3306
 DB_HOST_EXPDB=database:3306
 DB_USER_OPENML=root
@@ -10,4 +10,5 @@ DB_PASS_EXPDB_WRITE=ok
 DB_USER_EXPDB_READ=root
 DB_PASS_EXPDB_READ=ok
 ES_URL=elasticsearch:9200
-ES_PASSWORD=default
+ES_PASSWORD=default
+INDEX_ES_DURING_STARTUP=true

config/python/config

@@ -1,2 +1,2 @@
-apikey=AD000000000000000000000000000000
-server=http://nginx:80/api/v1/xml
+apikey=normaluser
+server=http://localhost:8000/api/v1/xml

docker-compose.yaml

@@ -1,6 +1,6 @@
 services:
   database:
-    image: "openml/test-database:20240105"
+    image: "openml/test-database:v0.1.20260204"
     container_name: "openml-test-database"
     environment:
       MYSQL_ROOT_PASSWORD: ok
@@ -54,11 +54,15 @@ services:
       context: config/nginx
     container_name: openml-nginx
     ports:
-      - "8000:80"
+      - "8000:8000"
+    networks:
+      default:
+        ipv4_address: 172.28.0.2
+
 
   php-api:
     profiles: ["all", "minio", "rest-api", "frontend", "evaluation-engine"]
-    image: openml/php-rest-api:v1.2.1
+    image: openml/php-rest-api:v1.2.4
     container_name: "openml-php-rest-api"
     ports:
       - "8080:80"  # also known as /api (nginx)
@@ -78,6 +82,8 @@ services:
       start_interval: 5s
       timeout: 3s
       interval: 1m
+    extra_hosts:
+      - "localhost=172.28.0.2"
 
   email-server:
     profiles: ["all", "frontend"]
@@ -95,7 +101,7 @@ services:
 
   frontend:
     profiles: ["all", "frontend"]
-    image: openml/frontend:dev_v2.0.20251111
+    image: openml/frontend:v2.1.1
     container_name: "openml-frontend"
     ports:
       - "8081:5000"  # also known as / (nginx)
@@ -108,7 +114,7 @@ services:
 
   minio:
     profiles: ["all", "minio", "evaluation-engine"]
-    image: openml/test-minio:v0.1.20241110
+    image: openml/test-minio:v0.1.20260204
     container_name: "openml-minio"
     ports:
       - "9000:9000"  # also known as /data (nginx)
@@ -133,6 +139,8 @@ services:
     depends_on:
       php-api:
         condition: service_healthy
+    extra_hosts:
+      - "localhost=172.28.0.2"
 
   croissants:
     profiles: ["all"]
@@ -157,7 +165,15 @@ services:
     depends_on:
       php-api:
         condition: service_healthy
+    extra_hosts:
+      - "localhost=172.28.0.2"
 
 networks:
   default:
     name: openml-services
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.28.0.0/16
+          ip_range: 172.28.1.0/24
+