Addapt data fodler to accepet data engineering ocnvention https://towardsdatascience.com/the-importance-of-layered-thinking-in-data-engineering-a09f685edc71
This commit is contained in:
parent
10f74be073
commit
19fe61b656
@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"project_name": "project_name",
|
"project_name": "project_name",
|
||||||
"repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_') }}",
|
"repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_') }}",
|
||||||
"package_name": "{{ cookiecutter.package_name.lower().replace(' ', '_') }}",
|
"package_name": "package_name",
|
||||||
"author_name": "Your name (or your organization/company/team)",
|
"author_name": "Your name (or your organization/company/team)",
|
||||||
"description": "A short description of the project.",
|
"description": "A short description of the project.",
|
||||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"],
|
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"],
|
||||||
|
@ -1,12 +1,10 @@
|
|||||||
.PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3
|
.PHONY: clean data lint requirements
|
||||||
|
|
||||||
#################################################################################
|
#################################################################################
|
||||||
# GLOBALS #
|
# GLOBALS #
|
||||||
#################################################################################
|
#################################################################################
|
||||||
|
|
||||||
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||||
BUCKET = {{ cookiecutter.s3_bucket }}
|
|
||||||
PROFILE = {{ cookiecutter.aws_profile }}
|
|
||||||
PROJECT_NAME = {{ cookiecutter.repo_name }}
|
PROJECT_NAME = {{ cookiecutter.repo_name }}
|
||||||
PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }}
|
PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }}
|
||||||
|
|
||||||
@ -38,22 +36,6 @@ clean:
|
|||||||
lint:
|
lint:
|
||||||
flake8 src
|
flake8 src
|
||||||
|
|
||||||
## Upload Data to S3
|
|
||||||
sync_data_to_s3:
|
|
||||||
ifeq (default,$(PROFILE))
|
|
||||||
aws s3 sync data/ s3://$(BUCKET)/data/
|
|
||||||
else
|
|
||||||
aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
|
||||||
endif
|
|
||||||
|
|
||||||
## Download Data from S3
|
|
||||||
sync_data_from_s3:
|
|
||||||
ifeq (default,$(PROFILE))
|
|
||||||
aws s3 sync s3://$(BUCKET)/data/ data/
|
|
||||||
else
|
|
||||||
aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
|
||||||
endif
|
|
||||||
|
|
||||||
## Set up python interpreter environment
|
## Set up python interpreter environment
|
||||||
create_environment:
|
create_environment:
|
||||||
ifeq (True,$(HAS_CONDA))
|
ifeq (True,$(HAS_CONDA))
|
||||||
|
@ -3,8 +3,3 @@ Commands
|
|||||||
|
|
||||||
The Makefile contains the central entry points for common tasks related to this project.
|
The Makefile contains the central entry points for common tasks related to this project.
|
||||||
|
|
||||||
Syncing data to S3
|
|
||||||
^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
* `make sync_data_to_s3` will use `aws s3 sync` to recursively sync files in `data/` up to `s3://{{ cookiecutter.s3_bucket }}/data/`.
|
|
||||||
* `make sync_data_from_s3` will use `aws s3 sync` to recursively sync files from `s3://{{ cookiecutter.s3_bucket }}/data/` to `data/`.
|
|
||||||
|
@ -1,6 +1,3 @@
|
|||||||
Getting started
|
Getting started
|
||||||
===============
|
===============
|
||||||
|
|
||||||
This is where you describe how to get set up on a clean install, including the
|
|
||||||
commands necessary to get the raw data (using the `sync_data_from_s3` command,
|
|
||||||
for example), and then how to make the cleaned, final data sets.
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user