Dive into Code

==[\d].*

import asyncio
from django.core.management.base import BaseCommand
from app.models import (
    Model1,
)

class Command(BaseCommand):

    async def import_data_async(self):
        item1, c1_created = await Model1.objects.aupdate_or_create(name="some name")
        item2, c1_created = await Model1.objects.aupdate_or_create(name="some name 2")
        await item1.related.aadd(item2)
 
    def handle(self, *args, **options):
        loop = asyncio.get_event_loop()
        loop.run_until_complete(self.import_data_async())
        self.stdout.write(self.style.SUCCESS('Data import completed successfully.'))

def cache_per_user(ttl=None, prefix=None, cache_post=False):
    '''Decorator that caches the view for each user
        * ttl - Cache lifetime, not sending this parameter means the
          cache will last until the server restarts or decides to remove it
        * prefix - Prefix to be used to cache the response. if not
          be informed will use 'view_cache_'+function.__name__
        * cache_post - Informs whether to cache POST requests
        * Cache for anonymous users is shared with everyone
        * The cache key will be one of the possible options:
            '%s_%s'%(prefix, user.id)
            '%s_anonymous'%(prefix)
           'view_cache_%s_%s_%s_%s'%(function.__name__, user.id,args,kwargs)
            'view_cache_%s_anonymous'%(function.__name__)
    '''
    def decorator(function):
        def apply_cache(request, *args, **kwargs):
            if request.user.is_anonymous():
                user = 'anonymous'
            else:
                user = request.user.id

            if prefix:
                base_key = '%s_%s' % (prefix, user)
            else:
                base_key = 'view_cache_%s_%s' % (function.__name__, user)

            # Include function arguments in the cache key
            args_key = '_'.join([str(arg) for arg in args])

            # Include keyword arguments in the cache key
            kwargs_key = '_'.join(['%s=%s' % (key, value) for key, value in kwargs.items()])

            # Generate the cache key
            CACHE_KEY = '%s_%s_%s' % (base_key, args_key, kwargs_key)

            if not cache_post and request.method == 'POST':
                can_cache = False
            else:
                can_cache = True

            if can_cache:
                response = cache.get(CACHE_KEY, None)
            else:
                response = None

            if not response:
                response = function(request, *args, **kwargs)
                if can_cache:
                    cache.set(CACHE_KEY, response, ttl)
            return response

        return apply_cache

    return decorator

SELECT pg_size_pretty( pg_database_size('database_name') );

import scrapy
import re
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from urllib.parse import urljoin, urlparse


# class ExternalLinkExtractor(LinkExtractor):
#     def extract_links(self, response):
#         links = super().extract_links(response)
#         allowed_domains = self.allow_domains
#         for link in links:
#             parsed_url = urlparse(link.url)
#             if parsed_url.netloc and parsed_url.netloc not in allowed_domains:
#                 yield link



class SearchSpider(CrawlSpider):
    name = "search"
    allowed_domains = ["example.com"]
    start_urls = ["https://example.com/index.html"]

    rules = (
        Rule(
            LinkExtractor(allow=()),
            callback="parse_item",
            follow=True
        ),
    )


    def parse_item(self, response):
        # get the content type of the response
        content_type = response.headers.get('Content-Type').decode('utf-8')
        # get the body of the response and remove whitespace
        body = re.sub(r'\s+', ' ', response.body.decode('utf-8')).strip()
        
        # the url of the page
        page_url = response.url
        
        # the referer of the page 
        referer = response.request.headers.get('Referer')
        # the domain of the response url
        domain = response.url.split('/')[2]
        
        # get all urls of the page 
        urls = response.xpath('//a/@href').extract()
        cleaned_urls =[]
        elements = []
        for url in urls:
            cleaned_url = urljoin(response.url, url)
            cleaned_urls.append(cleaned_url)
        
        # get all elements of the page 
        for element in response.xpath('//*'):
            element_attributes = []

            # get all attributes of an element
            for attr in element.attrib:
                attribute = {}
                attr_value = element.attrib[attr]
                if attr_value.strip() != '':
                    attribute[attr] = attr_value
                    element_attributes.append(attribute)
            element_text = element.get()
            element_name = element.xpath('name()').get()
            elements.append({
                'name': element_name,
                'value': element_text,
                'attributes': element_attributes
            })
        yield {
            'url':page_url,
            'domain':domain,
            'response':body,
            'urls':cleaned_urls,
            'content_type':content_type,
            'elements':elements,
            'referer': referer,

        }