Installation and Configuration
django-admin startproject cryptocurrencytrackingcd cryptocurrencytrackingdjango-admin startapp trackingAPI
pip install djangorestframework
INSTALLED_APPS = [ ... 'rest_framework', 'trackingAPI', ]
pip install Celery
sudo apt-get install rabbitmq-server
sudo systemctl enable rabbitmq-serversudo systemctl start rabbitmq-server
CELERY_BROKER_URL = 'amqp://localhost'
import osfrom celery import Celeryos.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cryptocurrencytracking.settings')app = Celery('cryptocurrencytracking')app.config_from_object('django.conf:settings', namespace='CELERY')app.autodiscover_tasks()
from .celery import app as celery_app__all__ = ['celery_app']
Creating model
from django.db import modelsclass Cryptocurrency(models.Model): cryptocurrency = models.CharField(max_length=100) price = models.CharField(max_length=100) market_cap = models.CharField(max_length=100) change = models.CharField(max_length=100) def __str__(self): return self.cryptocurrency
Crawling Cryptocurrency Data
pip install beautifulsoup4
# tasks.pyfrom time import sleepfrom celery import shared_taskfrom bs4 import BeautifulSoupfrom urllib.request import urlopen, Requestfrom .models import Cryptocurrency@shared_task# do some heavy stuffdef crawl_currency(): print('Crawling data and creating objects in database ..') req = Request('https://coinranking.com', headers={'User-Agent': 'Mozilla/5.0'}) html = urlopen(req).read() bs = BeautifulSoup(html, 'html.parser') # Find first 5 table rows rows = bs.find('tbody', class_="table__body").find_all('tr', class_="table__row")[0:5] for row in rows: cryptocurrency = row.find('span', class_="profile__name").get_text().strip().replace('\n', '') values = row.find_all('div', class_="valuta") price = values[0].get_text().strip().replace('\n', '') market_cap = values[1].get_text().strip().replace('\n', '') change = row.find('div', class_="change").find('span').get_text().strip().replace('\n', '') print({'cryptocurrency': cryptocurrency, 'price':price, 'market_cap':market_cap, 'change':change}) # Create object in database from crawled data Cryptocurrency.objects.create( cryptocurrency = cryptocurrency, price = price, market_cap = market_cap, change = change ) # Sleep 3 seconds to avoid any errors sleep(3)
#tasks.py@shared_taskdef update_currency(): print('Updating data ..') req = Request('https://coinranking.com', headers={'User-Agent': 'Mozilla/5.0'}) html = urlopen(req).read() bs = BeautifulSoup(html, 'html.parser') rows = bs.find('tbody', class_="table__body").find_all('tr', class_="table__row")[0:5] for row in rows: cryptocurrency = row.find('span', class_="profile__name").get_text().strip().replace('\n', '') values = row.find_all('div', class_="valuta") price = values[0].get_text().strip().replace('\n', '') market_cap = values[1].get_text().strip().replace('\n', '') change = row.find('div', class_="change").find('span').get_text().strip().replace('\n', '') print({'cryptocurrency': cryptocurrency, 'price':price, 'market_cap':market_cap, 'change':change}) data = {'cryptocurrency': cryptocurrency, 'price':price, 'market_cap':market_cap, 'change':change} Cryptocurrency.objects.filter(cryptocurrency=cryptocurrency).update(**data) sleep(3) # Run this function if database is emptyif not Cryptocurrency.objects: crawl_currency()while True: sleep(15) update_currency()
celery -A cryptocurrencytracking worker -l info
Building API
#serializers.pyfrom rest_framework import serializersfrom .models import Cryptocurrencyclass CryptocurrencySerializer(serializers.ModelSerializer): class Meta: model = Cryptocurrency fields = ['cryptocurrency', 'price', 'market_cap', 'change']
#views.pyfrom django.shortcuts import renderfrom rest_framework import genericsfrom .models import Cryptocurrencyfrom .serializers import CryptocurrencySerializerclass ListCryptocurrencyView(generics.ListAPIView): """ Provides a get method handler. """ queryset = Cryptocurrency.objects.all() serializer_class = CryptocurrencySerializer
#urls.pyfrom django.contrib import adminfrom django.urls import pathfrom trackingAPI.views import ListCryptocurrencyViewurlpatterns = [ path('admin/', admin.site.urls), path('', ListCryptocurrencyView.as_view()),]