gitlab runner config, realpath for macos in test/sys.t, remove unused function from py.h

Author: jhanna-kx

.gitlab-ci.yml

@@ -0,0 +1,104 @@
+#TODO test for w64/w32
+#TODO test for m64arm
+stages:
+- build
+- test
+- deploy
+- github
+
+variables:
+ GIT_DEPTH: 10
+workflow:
+ rules:
+ - if: '$CI_COMMIT_MESSAGE =~ /^release .*/'
+ when: never
+ - when: always
+
+trigger-builds:
+ stage: build
+ script: echo trigger build
+ tags:
+ - l64
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "push"'
+ when: manual
+
+build:
+ stage: build
+ tags:
+ - $platform
+ script:
+ - VER=$CI_COMMIT_TAG;[ "x$VER" = "x" ] && VER=$CI_COMMIT_BRANCH-$CI_COMMIT_SHA; export VER
+ - perl -p -i -e 's/EMBEDPYVERSION/`\$\"'${VER}'\"/g' p.q
+ - make $platform/p.so
+ artifacts:
+ expire_in: 2 days
+ paths:
+ - p.{k,q}
+ - $platform/p.so
+ parallel:
+ matrix:
+ - platform: [l64,m64] # TODO win
+ needs:
+ - trigger-builds
+
+test-artifacts:
+ stage: build
+ tags:
+ - l64
+ script: echo package-tests
+ artifacts:
+ expire_in: 2 days
+ paths:
+ - test.q
+ - tests/
+ - ci/makefile
+ needs:
+ - trigger-builds
+
+test:
+ stage: test
+ tags:
+ - $platform
+ variables:
+ GIT_STRATEGY: none
+ script: make -f ci/makefile test
+ dependencies:
+ - build
+ - test-artifacts
+ parallel:
+ matrix:
+ - platform: [l64,m64] # TODO m64arm
+deploy:
+ stage: deploy
+ tags:
+ - $platform
+ dependencies:
+ - build
+ - test
+ script: make -f ci/makefile $platform/$target
+ artifacts:
+ expire_in: 2 days
+ paths:
+ - embedPy-*-$CI_COMMIT_TAG.tgz
+ rules:
+ - if: '$CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""'
+ when: never
+ - when: manual
+ parallel:
+ matrix:
+ - platform: l64
+ target: [docker,github,conda]
+ - platform: m64
+ target: [github,conda]
+ghrelease:
+ stage: github
+ tags:
+ - l64
+ script: make -f ci/makefile l64/ghrelease
+ dependencies:
+ - deploy
+ rules:
+ - if: '$CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""'
+ when: never
+ - when: manual

README.md

@@ -1,6 +1,6 @@
 # embedPy
 
-[![GitHub release (latest by date)](https://img.shields.io/github/v/release/kxsystems/embedPy)](https://github.com/kxsystems/embedpy/releases) [![Travis (.org) branch](https://img.shields.io/travis/kxsystems/embedpy/master?label=travis%20build)](https://travis-ci.org/kxsystems/embedpy/branches) [![AppVeyor branch](https://img.shields.io/appveyor/ci/jhanna-kx/embedpy-ax90d/master?label=appveyor%20build)](https://ci.appveyor.com/project/jhanna-kx/embedpy-ax90d/branch/master)
+[![GitHub release (latest by date)](https://img.shields.io/github/v/release/kxsystems/embedPy)](https://github.com/kxsystems/embedpy/releases) [![AppVeyor branch](https://img.shields.io/appveyor/ci/jhanna-kx/embedpy-ax90d/master?label=appveyor%20build)](https://ci.appveyor.com/project/jhanna-kx/embedpy-ax90d/branch/master)
 
 Allows the kdb+ interpreter to manipulate Python objects and call Python functions.
 Part of the [_Fusion for kdb+_](https://code.kx.com/v2/interfaces/fusion/) interface collection.
@@ -13,17 +13,8 @@ Please [report issues](https://github.com/KxSystems/embedpy/issues) in this repo
 ## Requirements
 
 - kdb+ ≥ 3.5 64-bit/32-bit(Linux/Arm)
-- Python ≥ 3.5.0 (macOS/Linux/Arm) ≥ 3.6.0 windows
-- numpy < 1.22
-
-> :warning: **numpy version**: Due to a breaking change with the numpy C api conversions between Python and q arrays is unsupported.
->```
->q)np:.p.import[`numpy]
->q)np[`:__version__]`
->"1.22.1"
->q)np[`:arange][10]`
->foreign
->```
+- Python ≥ 3.8.0 (macOS/Linux/Arm) ≥ 3.8.0 windows
+
 
 ## Overview
 

appveyor.yml

@@ -3,9 +3,6 @@ image:
 - Visual Studio 2017
 artifacts:
 - path: embedPy_windows-$(appveyor_repo_tag_name).zip
-init:
- - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
-
 skip_non_tags: false
 platform: x64
 build_script:

ci/conda.sh

@@ -0,0 +1,18 @@
+P=$1;CNDENV=$2;CNDCHN=$3
+. "$P/miniconda/etc/profile.d/conda.sh"
+set -e
+conda create -n $CNDENV conda-build
+conda activate $CNDENV
+conda install -q -y anaconda-client
+EMBEDPY_VERSION=$CI_COMMIT_TAG
+export EMBEDPY_VERSION
+conda build conda-recipe --output -c $CNDCHN > packagenames.txt
+conda build -c $CNDCHN conda-recipe --no-long-test-prefix --no-include-recipe
+set +x
+CONDATOKEN=$(gpg -d --batch $P/condauploadtoken.gpg)
+for pack in $(cat packagenames.txt)
+do
+ anaconda -t $CONDATOKEN upload -l dev $pack
+done
+conda deactivate
+conda env remove -n $CNDENV

ci/makefile

@@ -0,0 +1,52 @@
+ifeq ($(platform),l64)
+ CNDPLAT := linux-64
+ CNDURL := https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+ RMRPATH := patchelf --remove-rpath
+else
+ CNDPLAT := osx-64
+ CNDURL := https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+ RMRPATH := install_name_tool -delete_rpath @loader_path/../../lib
+endif
+P ?= $(HOME)/.gitlab-runner
+KDBF ?= kdb-4.0-20221026.tar.bz2
+KDBURL := https://anaconda.org/kx/kdb/4.0/download/$(CNDPLAT)/$(KDBF)
+CNDENV := bld_$(platform)
+CNDCHN ?= kx
+#prereqs
+q: $(P)/q/$(platform)/q
+#strip conda rpaths
+$(P)/q/$(platform)/q:
+curl -O -J -L $(KDBURL)
+tar xf $(KDBF) q/q.k q/$(platform)/q
+mv q $(P)
+$(RMRPATH) $(P)/q/$(platform)/q
+
+conda: $(P)/miniconda/
+$(P)/miniconda/:
+wget $(CNDURL) -O miniconda.sh
+bash miniconda.sh -b -p $(P)/miniconda
+. "$(P)/miniconda/etc/profile.d/conda.sh" && \
+hash -r && \
+conda config --set always_yes yes --set changeps1 no && \
+conda update -q conda && \
+conda install -q conda-build
+#tests
+test: q
+pip3 install -r tests/requirements.txt
+cp $(platform)/p.so .
+QHOME=$(P)/q QLIC=$(P) $(P)/q/$(platform)/q test.q -s 4
+ci/test_virtualenv.sh
+
+#release and deploy
+l64/ghrelease:
+gpg -d --batch $(P)/ghtoken.gpg |gh auth login --with-token
+gh release create -d -p -R KxSystems/embedpy --generate-notes $(CI_COMMIT_TAG) embedPy-*-$(CI_COMMIT_TAG).tgz
+
+$(platform)/conda: conda
+ci/conda.sh $(P) $(CNDENV) $(CNDCHN)
+l64/docker:
+docker build -t kxsys/embedpy:latest -f docker/Dockerfile .
+docker tag kxsys/embedpy:latest kxsys/embedpy:$(CI_COMMIT_TAG)
+#echo $(DOCKER_PASSWORD)|docker login --username $(DOCKER_USERNAME) --password-stdin && docker push kxsys/embedpy:latest && docker push kxsys/embedpy:$(CI_COMMIT_TAG)
+$(platform)/github:
+tar czf embedPy-$(platform)-$(CI_COMMIT_TAG).tgz p.q p.k test.q tests $(platform)/p.so LICENSE README.md

ci/test_virtualenv.sh

@@ -0,0 +1,6 @@
+pip3 install virtualenv;
+virtualenv test_virtualenv;
+. test_virtualenv/bin/activate
+pip -q install -r tests/requirements.txt;
+q test.q -s 4 -q;
+

conda-recipe/conda_build_config.yaml

@@ -4,5 +4,6 @@ target_platform: # [osx]
  - osx-64 # [osx]
  - osx-arm64 # [osx]
 python:
+ - 3.8
  - 3.9
  - 3.10

conda-recipe/run_test.sh

@@ -12,4 +12,3 @@ then
 else
  echo cross compile, no tests
 fi
-

py.h

@@ -26,7 +26,6 @@ typedef struct _p _p,*P;struct _p{L r;P t;L n;union{P*p;P v[1];};};typedef struc
  X(V,Py_DecRef,(P))\
  X(V,Py_IncRef,(P))\
  X(V,PyErr_Clear,())\
- X(V,PyErr_Print,())\
  X(V,PyErr_Fetch,(P*,P*,P*))\
  X(V,PyErr_NormalizeException,(P*,P*,P*))\
  X(P,PyErr_BadArgument,())\

tests/sys.t

@@ -3,14 +3,15 @@ sysconfig:.p.import`sysconfig
 
 qPrefix:sys[`:prefix]`
 qBasePrefix:sys[`:base_prefix]`
+rp:{x[y]`}.p.import[`os;`:path.realpath]
 // Have to remove q path to coincide with python
-qPath:":" sv ((sys[`:path]`) except enlist getenv`QHOME)
+qPath:":" sv rp each ((sys[`:path]`) except enlist getenv`QHOME)
 qVersion:3#sys[`:version_info]`
 qLibdir:sysconfig[`:get_config_var]["LIBDIR"]`
 
 pyPrefix:raze system"python -c \"import sys; print(sys.prefix)\""
 pyBasePrefix:raze system"python -c \"import sys; print(sys.base_prefix)\""
-pyPath:raze system"python -c \"import sys; print(':'.join(sys.path))\""
+pyPath:":"sv rp each ":"vs raze system"python -c \"import sys; print(':'.join(sys.path))\""
 pyVersion:"J"$3#system"python -c \"import sys; [print(getattr(sys.version_info,val)) for val in ('major','minor','micro','releaselevel','serial')]\""
 pyLibdir:raze system"python -c \"import sysconfig; print(sysconfig.get_config_var('LIBDIR'))\""