storage(postgresql): add Postgres connector + config schema + dependency

- Introduce _PostgresConnector and wire backend='postgresql' in StorageBackend.create(). - Read connection parameters from site config: storage/0/postgresql_driver, postgresql_host, postgresql_port, postgresql_db, postgresql_conn_timeout and credentials from env: RFM_POSTGRES_USER / RFM_POSTGRES_PASSWORD. - Pass connect_timeout via DBAPI connect args. - Extend config schema (schemas/config.json) with the new Postgres options and add sensible defaults (driver=psycopg2, conn_timeout=60). - Keep SQLite as-is; no behavior change for existing users. - Add psycopg2-binary==2.9.8 to requirements.txt to provide the PG driver.

Author: mredenti

reframe/frontend/reporting/storage.py

@@ -34,15 +34,15 @@
 
 
 class _ConnectionStrategy:
- '''Abstract helper class for building SQLAlchemy engine configurations'''
+ '''Abstract helper class for building the URL and kwargs for a given SQL dialect'''
 
  def __init__(self):
  self.url = self._build_connection_url()
  self.engine = create_engine(self.url, **self._connection_kwargs)
 
  @abc.abstractmethod
  def _build_connection_url(self):
- '''Return a SQLAlchemy URL string for this backend.
+ '''Return a SQLAlchemy URL string for this dialect.
 
  Implementations must return a URL suitable for passing to 
  `sqlalchemy.create_engine()`.
@@ -74,6 +74,7 @@ def __init__(self):
  os.makedirs(prefix, exist_ok=True)
 
  open(self.__db_file, 'a').close()
+ # Update DB file mode
  os.chmod(self.__db_file, self.__db_file_mode)
 
  super().__init__()
@@ -98,6 +99,33 @@ def _connection_kwargs(self):
  return {'connect_args': {'timeout': timeout}}
 
 
+class _PostgresConnector(_ConnectionStrategy):
+ def __init__(self):
+ super().__init__()
+
+ def _build_connection_url(self):
+ host = runtime().get_option('storage/0/postgresql_host')
+ port = runtime().get_option('storage/0/postgresql_port')
+ db = runtime().get_option('storage/0/postgresql_db')
+ driver = runtime().get_option('storage/0/postgresql_driver')
+ user = os.getenv('RFM_POSTGRES_USER')
+ password = os.getenv('RFM_POSTGRES_PASSWORD')
+ if not (driver and host and port and db and user and password):
+ raise ReframeError(
+ 'Postgres connection info must be set in config and env')
+
+ return URL.create(
+ drivername=f'postgresql+{driver}',
+ username=user, password=password,
+ host=host, port=port, database=db
+ ).render_as_string(hide_password=False)
+
+ @property
+ def _connection_kwargs(self):
+ timeout = runtime().get_option('storage/0/postgresql_conn_timeout')
+ return {'connect_args': {'connect_timeout': timeout}}
+
+
 class StorageBackend:
  '''Abstract class that represents the results backend storage'''
 
@@ -106,6 +134,8 @@ def create(cls, backend, *args, **kwargs):
  '''Factory method for creating storage backends'''
  if backend == 'sqlite':
  return _SqlStorage(_SqliteConnector(), *args, **kwargs)
+ elif backend == 'postgresql':
+ return _SqlStorage(_PostgresConnector(), *args, **kwargs)
  else:
  raise ReframeError(f'no such storage backend: {backend}')
 

reframe/schemas/config.json

@@ -331,7 +331,7 @@
  "prepare_cmds": {
  "type": "array",
  "items": {"type": "string"}
-  },
+ },
  "processor": {"$ref": "#/defs/processor_info"},
  "devices": {"$ref": "#/defs/devices"},
  "features": {
@@ -560,6 +560,11 @@
  "sqlite_conn_timeout": {"type": "number"},
  "sqlite_db_file": {"type": "string"},
  "sqlite_db_file_mode": {"type": "string"},
+ "postgresql_driver": {"type": "string"},
+ "postgresql_host": {"type": "string"},
+ "postgresql_port": {"type": "number"},
+ "postgresql_db": {"type": "string"},
+ "postgresql_conn_timeout": {"type": "number"},
  "target_systems": {"$ref": "#/defs/system_ref"}
  }
  }
@@ -654,6 +659,8 @@
  "storage/sqlite_conn_timeout": 60,
  "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db",
  "storage/sqlite_db_file_mode": "644",
+ "storage/postgresql_conn_timeout": 60,
+ "storage/postgresql_driver": "psycopg2",
  "storage/target_systems": ["*"],
  "systems/descr": "",
  "systems/max_local_jobs": 8,

requirements.txt

@@ -36,4 +36,5 @@ tabulate==0.8.10; python_version == '3.6'
 tabulate==0.9.0; python_version >= '3.7'
 wcwidth==0.2.14
 sqlalchemy==2.0.41
+psycopg2-binary==2.9.8
 #+pygelf%pygelf==0.4.0