pydoop
A Python MapReduce and HDFS API for Hadoop
File Explorer
Download Latest Version (.zip)- hadoop_localfs.sh
- check_script_template.py
- run_checks
- start_container
- initialize.sh
- local_client_setup.sh
- initialize.sh
- local_client_setup.sh
- generate_conf_files.py
- zk_set.py
- zk_wait.py
- Dockerfile
- bootstrap.py
- create_hdfs_dirs.sh
- Dockerfile
- start_datanode.sh
- Dockerfile
- start_historyserver.sh
- Dockerfile
- start_namenode.sh
- Dockerfile
- start_nodemanager.sh
- Dockerfile
- start_resourcemanager.sh
- Dockerfile
- start_namenode.sh
- Dockerfile
- docker-compose.yml
- Dockerfile
- Dockerfile
- build_base_images.sh
- build_cluster_images.sh
- share_etc_hosts.py
- start_client.sh
- start_cluster.sh
- cluster.rst
- build_deprecation_tables
- bump_copyright_year
- docker_build
- dump_app_params
- edit_conf
- git_export
- import_src
- mapred_pipes
- unpack_debian
- update_docs
- .gitignore
- crs4.png
- layout.html
- hadut.rst
- hdfs_api.rst
- index.rst
- mr_api.rst
- avro.rst
- index.rst
- input_format.rst
- intro.rst
- sequence_file.rst
- archive.rst
- index.rst
- latest.rst
- hdfs_api.rst
- index.rst
- mapred_api.rst
- pydoop_script.rst
- conf.py
- how_to_cite.rst
- index.rst
- installation.rst
- Makefile
- pydoop_script.rst
- pydoop_script_options.rst
- pydoop_submit_options.rst
- running_pydoop_applications.rst
- self_contained.rst
- mini_aligned_seqs.gz.parquet
- avro_base.py
- avro_container_dump_results.py
- avro_key_in.py
- avro_key_in_out.py
- avro_key_value_in.py
- avro_key_value_in_out.py
- avro_parquet_dump_results.py
- avro_pyrw.py
- avro_value_in.py
- avro_value_in_out.py
- check_cc.py
- check_results.py
- color_count.py
- create_input.py
- gen_data.py
- generate_avro_users.py
- kmer_count.py
- show_kmer_count.py
- write_avro.py
- alignment_record.avsc
- alignment_record_proj.avsc
- pet.avsc
- stats.avsc
- user.avsc
- WriteKV.java
- WriteParquet.java
- build.sh
- config.sh
- pom.xml
- run
- run_avro_container_in
- run_avro_container_in_out
- run_avro_parquet_in
- run_avro_parquet_in_out
- run_avro_pyrw
- run_color_count
- run_kmer_count
- write_avro_kv
- Pipes.hh
- SerialUtils.hh
- StringUtils.hh
- TemplateFactory.hh
- HadoopPipes.cc
- Makefile
- README.txt
- SerialUtils.cc
- StringUtils.cc
- wordcount.cc
- common.py
- repl_session.py
- run
- treegen.py
- treewalk.py
- alice_1.txt
- alice_2.txt
- TextInputFormat.java
- TextInputFormat.java
- check_results.py
- run
- example_1.sam
- example_2.sam
- matrix.txt
- stop_words.txt
- base_histogram.py
- caseswitch.py
- grep.py
- lowercase.py
- transpose.py
- wc_combiner.py
- wordcount.py
- wordcount_sw.py
- check.py
- run
- run_script.sh
- cols_1.txt
- cols_2.txt
- map_only_java_writer.py
- map_only_python_writer.py
- nosep.py
- wordcount_full.py
- wordcount_minimal.py
- check.py
- run
- run_submit.sh
- __init__.py
- __init__.py
- main.py
- mapper.py
- reducer.py
- __init__.py
- check_results.py
- run
- filter.py
- wordcount.py
- check.py
- run
- config.sh
- README
- run_all
- f1.txt
- f2.txt
- f1.txt
- f2.txt
- f.txt
- map_only_java_writer.py
- map_only_python_writer.py
- map_reduce_combiner.py
- map_reduce_java_rw.py
- map_reduce_java_rw_pstats.py
- map_reduce_python_partitioner.py
- map_reduce_python_reader.py
- map_reduce_python_writer.py
- map_reduce_raw_io.py
- map_reduce_slow_java_rw.py
- map_reduce_slow_python_rw.py
- check.py
- genwords.py
- run
- run_app.sh
- run_perf.sh
- check.py
- gen_splits.py
- mrapp.py
- run
- mrapp.py
- run
- config.sh
- run_all
- avro-mapred-1.7.7-hadoop2.jar
- favicon.svg
- logo.svg
- ubuntu-font-family.tar.bz2
- __init__.py
- argparse_types.py
- main.py
- script.py
- script_template.py
- submit.py
- __init__.py
- __init__.py
- common.py
- file.py
- fs.py
- path.py
- __init__.py
- api.py
- binary_protocol.py
- connections.py
- pipes.py
- __init__.py
- conversion_tables.py
- jvm.py
- misc.py
- py3compat.py
- __init__.py
- avrolib.py
- hadoop_utils.py
- hadut.py
- jc.py
- test_support.py
- test_utils.py
- Application.java
- BinaryProtocol.java
- DownwardProtocol.java
- DummyRecordReader.java
- OpaqueSplit.java
- OutputHandler.java
- PipesMapper.java
- PipesNonJavaInputFormat.java
- PipesNonJavaOutputFormat.java
- PipesPartitioner.java
- PipesReducer.java
- PydoopAvroBridgeKeyReader.java
- PydoopAvroBridgeKeyValueReader.java
- PydoopAvroBridgeKeyValueWriter.java
- PydoopAvroBridgeKeyWriter.java
- PydoopAvroBridgeReaderBase.java
- PydoopAvroBridgeValueReader.java
- PydoopAvroBridgeValueWriter.java
- PydoopAvroBridgeWriterBase.java
- PydoopAvroInputBridgeBase.java
- PydoopAvroInputKeyBridge.java
- PydoopAvroInputKeyValueBridge.java
- PydoopAvroInputValueBridge.java
- PydoopAvroKeyInputFormat.java
- PydoopAvroKeyOutputFormat.java
- PydoopAvroKeyRecordReader.java
- PydoopAvroKeyRecordWriter.java
- PydoopAvroKeyValueInputFormat.java
- PydoopAvroKeyValueOutputFormat.java
- PydoopAvroKeyValueRecordReader.java
- PydoopAvroKeyValueRecordWriter.java
- PydoopAvroOutputBridgeBase.java
- PydoopAvroOutputFormatBase.java
- PydoopAvroOutputKeyBridge.java
- PydoopAvroOutputKeyValueBridge.java
- PydoopAvroOutputValueBridge.java
- PydoopAvroRecordReaderBase.java
- PydoopAvroRecordWriterBase.java
- PydoopAvroValueInputFormat.java
- PydoopAvroValueOutputFormat.java
- PydoopAvroValueRecordReader.java
- PydoopAvroValueRecordWriter.java
- Submitter.java
- TaskLog.java
- TaskLogAppender.java
- UpwardProtocol.java
- NoSeparatorTextOutputFormat.java
- htable.c
- htable.h
- hdfs.h
- mutexes.c
- platform.h
- thread.c
- thread_local_storage.c
- inttypes.h
- mutexes.c
- platform.h
- thread.c
- thread_local_storage.c
- unistd.h
- mutexes.h
- thread.h
- thread_local_storage.h
- config.h
- exception.c
- exception.h
- hdfs.c
- jni_helper.c
- jni_helper.h
- hdfs_file.cc
- hdfs_file.h
- hdfs_fs.cc
- hdfs_fs.h
- hdfs_module.cc
- SerialUtils.cc
- SerialUtils.hh
- hu_extras.cpp
- hu_extras.h
- sercore.cpp
- streams.cpp
- streams.h
- buf_macros.h
- py3k_compat.h
- Py_macros.h
- __init__.py
- all_tests.py
- test_submit.py
- all_tests.py
- common.py
- test_io.py
- user.avsc
- __init__.py
- all_tests.py
- test_hadoop_utils.py
- test_hadut.py
- test_pydoop.py
- test_test_support.py
- __init__.py
- all_tests.py
- common_hdfs_tests.py
- test_common.py
- test_core.py
- test_hdfs.py
- test_hdfs_fs.py
- test_local_fs.py
- test_path.py
- try_hdfs.py
- OpaqueRoundtrip.java
- __init__.py
- all_tests.py
- m_task.cmd
- r_task.cmd
- test_connections.py
- test_opaque.py
- all_tests.py
- test_deser.py
- test_streams.py
- __init__.py
- all_tests.py
- .dir-locals.el
- .dockerignore
- .gitignore
- .travis.yml
- AUTHORS
- Dockerfile
- Dockerfile.client
- Dockerfile.docs
- LICENSE
- MANIFEST.in
- notice_template.txt
- pydoop.properties
- README.md
- requirements.txt
- setup.cfg
- setup.py
- VERSION
# Installation Guide
1. Get the code
git clone https://github.com/crs4/pydoop
Downloads the entire project code from GitHub to your computer.
cd pydoop
Moves into the project folder you just downloaded.
2. Docker
Easy RecommendedPrerequisites
- Git Needed to download the project code from GitHub.
- Docker Desktop Needed to build and run containers. Install it and keep it running in the background.
docker compose -f dev_tools/docker/clusters/apache_2.6.0/docker-compose.yml up -d --build
Runs the command against the services defined in the compose file.
Run docker compose ps to check the containers are Up. If the README mentions a port, open http://localhost:PORT in your browser.
3. Python
EasyPrerequisites
pip install -r requirements.txt
Installs the Python libraries listed in requirements.txt (or similar).
python <์คํํ ํ์ผ๋ช
>.py # README์์ ์ ํํ ์คํ ํ์ผ๋ช
์ ํ์ธํ์ธ์
Runs the Python script (or module).
If it runs without errors and prints output in the terminal, it worked.
// repository documentation
Was this content helpful?
(0 ratings)