SKA-ScienceDataProcessor · timcornwell · Apr 30, 2018 · Apr 25, 2018 · Apr 26, 2018 · Apr 26, 2018
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,12 @@ __pycache__
 /algorithm_reference_library.egg-info/
 
 **/*.class
+dask-worker-space
+/examples/arl/results
+/test_results
+/docs/libs
+/docs/processing_components
+/docs/workflows
+*/notebooks/*_files
+*/notebooks/*.rst
+*.log
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,18 @@
+
+Effective April 30 2018, the ARL has been updated to be consistent with the SDP Processing Architecture. This required 
+very substantial changes throughout. The code is consistent internally but ARL code kept outside the code tree will 
+need to be updated manually.
+
+* The top level directory arl has been split into three: libs, processing_components, and workflows
+    - libs contains functions that are not accessed directly by the Execution Framework
+    - processing_components contains functions that may be accessed by the EF. 
+    - workflows contains high level workflows using the processing_components. This eventually will migrate to the EF
+     but some are kept here as scripts or notebooks.
+* The tests and notebooks have been moved to be inside the appropriate directory.
+* The data definitions formerly in arl/data have been moved to a top level directory data_models. 
+* The top level Makefile has been updated
+* The docs have been updated
+* The use of the term 'graph' has been replaced in many places by 'list' to reflect the wrapping of dask in 
+arlexecute.
+
+
diff --git a/Makefile b/Makefile
@@ -25,13 +25,19 @@ build: in
 
 # clean out the cache before tests are run
 cleantests:
-	cd tests && rm -rf __pycache__
+	cd libs/tests && rm -rf __pycache__
+	cd processing_components/tests && rm -rf __pycache__
+	cd workflows/tests && rm -rf __pycache__
 
 unittest: cleantests
-	$(PYTHON) -m unittest discover -f --locals -s tests -p "test_*.py"
+	$(PYTHON) -m unittest discover -f --locals -s libs/tests -p "test_*.py"
+	$(PYTHON) -m unittest discover -f --locals -s processing_components/tests -p "test_*.py"
+	$(PYTHON) -m unittest discover -f --locals -s workflows/tests -p "test_*.py"
 
 pytest: cleantests
-	pytest -x -v tests/
+	pytest -x -v libs/tests/
+	pytest -x -v processing_components/tests/
+	pytest -x -v workflows/tests/
 
 nosetests: cleantests
 	rm -f predict_facet_timeslice_graph_wprojection.png pipelines-timings_*.csv
@@ -42,8 +48,12 @@ nosetests-coverage: inplace cleantests
 	$(NOSETESTS) -s -v --with-coverage arl
 
 trailing-spaces:
-	find arl -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
-	find tests -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find libs -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find libs/tests -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find processing_components -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find processing_components/tests -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find workflows -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
+	find workflows/tests -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
 
 docs: inplace
 	# you must have graphviz installed
@@ -61,7 +71,9 @@ code-lint:
 code-analysis: code-flake code-lint
 
 examples: inplace
-	$(MAKE) -C examples/arl
+	$(MAKE) -C libs/notebooks
+	$(MAKE) -C processing_components/notebooks
+	$(MAKE) -C workflows/notebooks
 
 notebook:
 	DEVICE=`ip link | grep -E " ens| wlan| eth" | grep BROADCAST | tail -1 | cut -d : -f 2  | sed "s/ //"` && \

diff --git a/README.md b/README.md
@@ -2,7 +2,8 @@
 Algorithm Reference Library
 ===========================
 
-This is a project to create a reference code in NumPy for aperture synthesis imaging.
+This is a project to create a reference code in NumPy for aperture synthesis 
+imaging. It is structured to match the SDP processing architecture.
 
 Motivation
 ----------

diff --git a/arl/__init__.py b/arl/__init__.py
diff --git a/arl/calibration/calibration.py b/arl/calibration/calibration.py
diff --git a/arl/data/persist.py b/arl/data/persist.py
diff --git a/arl/data/skymodel.py b/arl/data/skymodel.py
diff --git a/arl/graphs/dask_minimal.py b/arl/graphs/dask_minimal.py