Addapt data fodler to accepet data engineering ocnvention https://towardsdatascience.com/the-importance-of-layered-thinking-in-data-engineering-a09f685edc71
This commit is contained in:
parent
10f74be073
commit
19fe61b656
@ -1,7 +1,7 @@
|
||||
{
|
||||
"project_name": "project_name",
|
||||
"repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_') }}",
|
||||
"package_name": "{{ cookiecutter.package_name.lower().replace(' ', '_') }}",
|
||||
"package_name": "package_name",
|
||||
"author_name": "Your name (or your organization/company/team)",
|
||||
"description": "A short description of the project.",
|
||||
"open_source_license": ["MIT", "BSD-3-Clause", "No license file"],
|
||||
|
@ -1,12 +1,10 @@
|
||||
.PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3
|
||||
.PHONY: clean data lint requirements
|
||||
|
||||
#################################################################################
|
||||
# GLOBALS #
|
||||
#################################################################################
|
||||
|
||||
PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||
BUCKET = {{ cookiecutter.s3_bucket }}
|
||||
PROFILE = {{ cookiecutter.aws_profile }}
|
||||
PROJECT_NAME = {{ cookiecutter.repo_name }}
|
||||
PYTHON_INTERPRETER = {{ cookiecutter.python_interpreter }}
|
||||
|
||||
@ -38,22 +36,6 @@ clean:
|
||||
lint:
|
||||
flake8 src
|
||||
|
||||
## Upload Data to S3
|
||||
sync_data_to_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/
|
||||
else
|
||||
aws s3 sync data/ s3://$(BUCKET)/data/ --profile $(PROFILE)
|
||||
endif
|
||||
|
||||
## Download Data from S3
|
||||
sync_data_from_s3:
|
||||
ifeq (default,$(PROFILE))
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/
|
||||
else
|
||||
aws s3 sync s3://$(BUCKET)/data/ data/ --profile $(PROFILE)
|
||||
endif
|
||||
|
||||
## Set up python interpreter environment
|
||||
create_environment:
|
||||
ifeq (True,$(HAS_CONDA))
|
||||
|
@ -3,8 +3,3 @@ Commands
|
||||
|
||||
The Makefile contains the central entry points for common tasks related to this project.
|
||||
|
||||
Syncing data to S3
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
* `make sync_data_to_s3` will use `aws s3 sync` to recursively sync files in `data/` up to `s3://{{ cookiecutter.s3_bucket }}/data/`.
|
||||
* `make sync_data_from_s3` will use `aws s3 sync` to recursively sync files from `s3://{{ cookiecutter.s3_bucket }}/data/` to `data/`.
|
||||
|
@ -1,6 +1,3 @@
|
||||
Getting started
|
||||
===============
|
||||
|
||||
This is where you describe how to get set up on a clean install, including the
|
||||
commands necessary to get the raw data (using the `sync_data_from_s3` command,
|
||||
for example), and then how to make the cleaned, final data sets.
|
||||
|
Loading…
x
Reference in New Issue
Block a user