python-microscopy · zacsimile · Apr 8, 2022 · Apr 8, 2022 · Apr 8, 2022 · Apr 8, 2022
diff --git a/.github/workflows/make_release.yml b/.github/workflows/make_release.yml
@@ -47,4 +47,18 @@ jobs:
         -H "Accept: application/vnd.github.v3+json" \
         -u ${{ secrets.REC_REPO_TOKEN }}\
         -d '{"ref":"ref"}'
-
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Build and push
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        push: true
+        tags: python-microscopy/pyme-server:latest
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,99 @@
+# https://pythonspeed.com/articles/activate-conda-dockerfile
+# FROM continuumio/miniconda3
+FROM nvidia/cuda:11.4.0-devel-ubuntu20.04
+
+# create a working directory for docker
+WORKDIR /app
+
+# SHELL ["/bin/bash", "--login", "-c"]
+
+# >>> https://hub.docker.com/r/continuumio/miniconda3/dockerfile
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PATH /opt/conda/bin:$PATH
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y wget bzip2 ca-certificates curl git build-essential && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* 
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py39_4.11.0-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean -tipsy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc 
+    # && \ echo "conda activate base" >> ~/.bashrc
+
+# ENV TINI_VERSION v0.16.1
+# ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+# RUN chmod +x /usr/bin/tini
+
+# ENTRYPOINT [ "/usr/bin/tini", "--" ]
+# CMD [ "/bin/bash" ]
+# <<<
+
+# SHELL [ "/usr/bin/tini", "--", "/bin/bash", "--login", "-c"]
+
+# pull additional dependencies not included in continuumio/miniconda3
+# in theory, we could eliminate these if we get conda packaging right
+# - build-essential for python setup.py install
+# - freeglut3-dev for pyopengl
+# - libgtk2.0-0 for wx (NOTE: we also need the glib package 
+#   in docker-env.yaml for wx)
+# RUN apt-get update && \
+#     apt-get -y install build-essential && \
+#     apt-get -y install freeglut3-dev && \
+#     apt-get -y install libgtk2.0-0
+# RUN apt-get update && apt-get -y install build-essential
+
+# create environment
+COPY docker-env.yaml .
+RUN conda env create -f docker-env.yaml
+#RUN conda config --add channels david_baddeley
+#RUN conda create -n pyme python=3.7 pyme-depends python-microscopy
+
+# make run commands use the new environment
+RUN echo "conda activate pyme" >> ~/.bashrc
+# RUN conda activate pyme
+SHELL ["conda", "run", "-n", "pyme", "/bin/bash", "--login", "-c"]
+
+RUN pip install pycuda
+
+# ===> Ideally, swap these for proper conda builds
+
+# clone python-microscopy and pymecompress into workdir
+# RUN git clone https://github.com/python-microscopy/python-microscopy.git
+RUN git clone https://github.com/zacsimile/python-microscopy.git
+RUN git clone https://github.com/python-microscopy/pymecompress.git
+# RUN git clone https://github.com/inducer/pycuda.git
+RUN git clone https://github.com/python-microscopy/pyme-warp-drive.git
+
+# install pymecompress
+RUN cd pymecompress && python setup.py develop
+
+# install PYME development version
+RUN cd python-microscopy && git checkout dockerize && python setup.py develop
+
+# install pycuda
+# RUN cd pycuda && python setup.py develop
+
+# install pyme-warp-drive
+RUN cd pyme-warp-drive && python setup.py develop
+
+# make sure it worked, Docker build will crash here if not
+# in theory, we could and should eliminate this (it adds
+# unnecessary layers to the Docker image)
+# RUN echo "test for pyme installation"
+# RUN python -c "import PYME"
+
+# make dataserver root
+RUN mkdir ~/.PYME
+RUN echo $'dataserver-root: "/"\ndataserver-filter: ""' >> ~/.PYME/config.yaml
+
+# launch cluster
+ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "pyme", "PYMECluster"]
+
+# A shell entrypoint may be safer long term...
+# COPY server-entrypoint.sh .
+# RUN chmod +x server-entrypoint.sh
+# ENTRYPOINT ["./server-entrypoint.sh"]
diff --git a/PYME/cluster/launch_cluster.py b/PYME/cluster/launch_cluster.py
@@ -0,0 +1,148 @@
+# from PYME.misc import big_sur_fix
+import subprocess
+import time
+import sys
+import os
+import webbrowser
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+class ClusterNode(object):
+    def __init__(self, root_dir):
+        self._data_server = None
+        self._rule_server = None
+        self._node_server = None
+        self._cluster_ui = None
+
+        self._root_dir = root_dir
+
+    def _kill_procs(self, procs):
+        #ask nicely
+        for p in procs:
+            if not p is None:
+                p.send_signal(1)
+
+        #give the processes a chance to close
+        time.sleep(2)
+
+        #kill off stragglers
+        for p in procs:
+            if not p is None:
+                p.kill()
+
+        time.sleep(1)
+
+
+    def _launch_data_server(self):
+        if not self._data_server is None:
+            self._kill_procs([self._data_server,])
+
+        logger.info('Launching data server: root=%s' % self._root_dir)
+        self._data_server = subprocess.Popen('"%s" -m PYME.cluster.HTTPDataServer -a local -p 0 -r "%s"' % (sys.executable, self._root_dir), shell=True)
+
+    def _launch_rule_server(self):
+        if not self._rule_server is None:
+            self._kill_procs([self._rule_server, ])
+
+        logger.info('Launching rule server')
+        self._rule_server = subprocess.Popen('"%s" -m PYME.cluster.PYMERuleServer -a local -p 0'
+                                             '' % sys.executable, shell=True)
+
+    def _launch_node_server(self):
+        if not self._node_server is None:
+            self._kill_procs([self._node_server, ])
+
+        logger.info('Launching node server')
+        self._node_server = subprocess.Popen('"%s" -m PYME.cluster.PYMERuleNodeServer -a local -p 0' % sys.executable, shell=True)
+
+    def _launch_cluster_ui(self, gui=False):
+        try:
+            import django
+        except ImportError:
+            logger.error('django is not installed, to use clusterUI install django (2.0.x, 2.1.x)')
+
+        if not self._cluster_ui is None:
+            self._kill_procs([self._cluster_ui, ])
+
+        logger.info('Launching clusterUI')
+        self._cluster_ui_stderr = open('clusterui.log', 'w')
+        self._cluster_ui = subprocess.Popen('"%s" %s runserver 9999' % (sys.executable, os.path.join(os.path.split(__file__)[0], 'clusterUI', 'manage.py')), stderr=self._cluster_ui_stderr, shell=True)
+
+        if gui:
+            #launch a web-browser to view clusterUI
+            time.sleep(5)
+            webbrowser.open_new_tab('http://127.0.0.1:9999/')
+
+    def _launch_ruleserver_ui(self):
+        from PYME.misc import sqlite_ns
+        from . import distribution
+        ns = sqlite_ns.getNS('_pyme-taskdist')
+        ruleservers = distribution.getDistributorInfo(ns)
+        webbrowser.open_new_tab(list(ruleservers.values())[0])
+
+
+    def shutdown(self):
+        logger.info('Shutting down cluster')
+        print('Shutting down cluster')
+        self._kill_procs([self._node_server, self._rule_server, self._cluster_ui, self._data_server])
+        try:
+            self._cluster_ui_stderr.close()
+        except:
+            pass
+
+
+    def launch(self, gui=False, clusterUI=True, main_node=False):
+        self._launch_data_server()
+        if main_node:
+            self._launch_rule_server()
+
+            #wait for the rule server to come up before launching the node server
+            time.sleep(5)
+        self._launch_node_server()
+        if clusterUI:
+            self._launch_cluster_ui(gui=gui)
+        elif gui:
+            self._launch_ruleserver_ui()
+
+
+
+    def run(self, **kwargs):
+        # runs a busy loop monitoring status
+        self.launch(**kwargs)
+
+        try:
+            while True:
+                time.sleep(30)
+
+                #TODO - poll the processes to see if they are still running
+        finally:
+            self.shutdown()
+
+
+def main():
+    import PYME.resources
+    from PYME import config
+    from optparse import OptionParser
+    from PYME.IO.FileUtils import nameUtils
+
+    op = OptionParser(usage='usage: %s [options]' % sys.argv[0])
+    default_root = config.get('dataserver-root')
+    op.add_option('-r', '--root', dest='root',
+                  help="Root directory of virtual filesystem (default %s, see also 'dataserver-root' config entry)" % default_root,
+                  default=default_root)
+    op.add_option('--ui', dest='ui', help='launch web based ui', default=False)
+    op.add_option('--clusterUI', dest='clusterui', help='launch the full django-based cluster UI', 
+                  action='store_true', default=False)
+    op.add_option('--main', dest='main_node', help='Launch this as the main node for a cluster.',
+                  default=False, action='store_true')
+
+    options, args = op.parse_args()
+
+    cluster = ClusterNode(root_dir=options.root)
+
+    cluster.run(gui=options.ui, clusterUI=options.clusterui, main_node=options.main_node)
+
+if __name__ == '__main__':
+    main()
diff --git a/conda-recipes/python-microscopy/entry_points.yaml b/conda-recipes/python-microscopy/entry_points.yaml
@@ -14,6 +14,7 @@ entry_points:
     - PYMENodeServer = PYME.cluster.PYMENodeServer:main
     - PYMERuleNodeServer = PYME.cluster.PYMERuleNodeServer:main
     - PYMEClusterOfOne = PYME.cluster.cluster_of_one:main
+    - PYMECluster = PYME.cluster.launch_cluster:main
     - PYMEWebDav = PYME.cluster.webdav:main
     - PYMEscmosmapgen = PYME.Analysis.gen_sCMOS_maps:main
     - fitMonP = PYME.ParallelTasks.fitMonP:main

diff --git a/docker-env.yaml b/docker-env.yaml
@@ -0,0 +1,47 @@
+name: pyme
+channels:
+  - david_baddeley
+  - anaconda
+  # - conda-forge
+dependencies:
+  - python=3.7
+  # - pyme-depends
+  # - python-microscopy
+  - numpy>=1.21
+  - scipy>=1.7
+  - matplotlib # <=3.2.2
+  # - wxpython<4.1
+  - pytables
+  # - pyopengl
+  # - traits
+  # - traitsui<=7.1.0
+  # - pyface<=7.1.0
+  - pillow
+
+  - toposort
+  - networkx
+
+  - pyfftw
+  - mpld3
+  - cherrypy
+  - scikit-image
+  - scikit-learn
+  - zeroconf<=0.26.3
+  - requests
+  - pandas
+  - pyyaml
+  - psutil
+  - docutils
+  - sphinx
+  - ujson>=3.0.0
+  - jinja2
+  - pycairo
+  # - pymecompress>=0.2.0
+  - six
+  - future
+  - cython
+  - blosc<=1.19
+
+  - glib
+  - django=2.1
+  # - pycuda