From a6bd31ba67f175089c668d33bf1fcdb3dd02c0cc Mon Sep 17 00:00:00 2001 From: alexwholland Date: Sun, 19 Mar 2023 17:37:15 -0700 Subject: [PATCH] Setup database to store report metrics in search history --- db.sqlite3 | Bin 0 -> 135168 bytes scraper/templates/scraper/index.html | 14 +++-- scraper/views.py | 85 +++++++++++++++------------ 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/db.sqlite3 b/db.sqlite3 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c226ef11929c66dc94865f92dae5550157c9a000 100644 GIT binary patch literal 135168 zcmeI5TWlNIdB-^-Me0V5Y+0tgwxbcf@|r74n&EJG*^RqP+qA8q>z9qDF054RWvwa5o^cPyd0rO; zkH>S8{F^5KI-m1o$JaR^|8utejdo9Z=5N0-NR9a)QJXOTqu{a8{~0|o`uX9n1%5F6 zd;TvD7Ja|r{ocTj2d;7-_k49DN83H$dToCo{9ul2HH=C@f1<56@2RT|y{;DP)zw;~ zy>ArShbq}g$7My#i+OdmVmx%3Pn=!OW^QJMTT6@AZ)JtWrE}RkLd0pch0};|b*Vp7 zL^v5?!i+-Xln`O^h@MEa^~%0LxR~ZznpJ~#YE7?~jYh+$RvN0cPb-m%=aqaqrZZ)I z#{11$*yZ;bZQ#)+zAp={LXQq2es(+%ev62|vYGg{C!#wi#l(DWCGDCRwp};UrFz_u%DxyP zO*7lmG&YlbwbIlpO|`jJ(`lEAt;CZFJ)UDtW7+X8ZDOUW8+ozmwybIYabzGeZv+G3 zwOMYRx0=Mt(pNyfTF@(6Syx3#%gISavs%N-723ENETe6@-gZq$3v|RqqU(3Y0^v-C zd-iHm%az*KDZBdF#uCx6#ujHv(5%GG%$dts`*`9sq*%#7A{eAWi+WvHx_VPsx^?-o zaBX?F*F2d$*VhUAHT8M7SZuG1=(;!>2+z%N&n6sb*%f93+CCoJ2xB{WSHWwK z5!-7OX1h&fPhTDmgr}yscaHZW>vq184In)iotoBYJg(MBk9?%n^Y^s+Nhy|!cDp6% zBqgoUR7=&OQHclzt*JMSvTjc|8fs&ebPU=_BSNlPE!k7)Svmw*{$@^QdW#Q zDQ08Kj*v|)^s;7@tdf$7RT9Zhg*LU7m41;VMawshM?|GlKdrHPziL!;f~c?hQE9{< z&Q5V2HXJq{mdNdzN*apTllvRGXWrmQ-QCgkrJ+DLo#vh;t@dsACF|;HUl@I7hh18} z&*-_jLEFoYcVhI4{Ip7ap znBKHow28O-1=}AYchU5pXZ^*&KzMeRd-_XOeJqNsQr4m0rDj&@4pdLERgW~M^Oj)q z1})szD#dEgm$xZt>o2G7_p>!Ji8d|K))-pCj=ky#`N_ zoYB@@E)ZT=;MO%eyRuQNYfU;CGr3qN273HP`qH0^8sTDn@5GeeFP`4l=;S2n9=U0z=SU{g$k#P8aT7%`q1N?s^$`j5 zHubVv*6I)RCYd|O#F#uM%5zd$h{>-g(ko(mUQET3Vpl&X4ZH*l};*?B*%n}d=ms*C#uJ(IUHbly$C#!SsDBx?n&NHaZAOM0c)yyselSXxZTua6T?7doCK zb+yK#OSO9S$(n72r{V5FB;e2T#KV=22hr5h4ka)aTgZ@ZNpUGNN}MfroY8CP;i^&BJ242lg3?gQB@%O^ET)uHEOBOpxVY4Dq1fwX z1w+-!YRUET5>s(Sj=w%ke017cjN5&5*;H#y_lhWTLKM#q5f7cV7CYAp>vt^9xdmD= zo>FAxVt_d5bY8L3d945ui@0v-NtERHMgO2@=0e7d|1gY{VrQ)Ci3`ylT9f>X&|;*QCCV@QO)10l1tm2k(7)n$&8P+z6kZM7K#v zBx6bzW(a~>t`0wz4&Hq0C z4zKYEel+y$(4U1q7W#!yB6KMD-@&g2|1kJl!FupoFcu7qeSPfHV^7Fi0rO*ik`W&e z009sH0T2KI5C8!X0D)aZAaZoj!_CYE9vS*$)tqOX)hYr#7R=)lM~F>S3XnJ0b@JyW zY*y_vUAq_BcI2;kREj-z?K2%a;V`i~J{zD<)!Uqr!;Zx)c_oFLOZu5A+a#x}%seO1 z$G{iJ+b5j#s^667FgGRkWFT*z&q@2v0(pmo)O^sC+DSJxG7Iv82(>V^bu3I#^za4pItX`cW{`^MBvTc1{{!R& z4{kc@W2Lj1U$=O66d|u~kkfV?e0 z3`{W{15*e+JWSpW7(O;bC3I40@pPZA{~ry0!^8g<|1JLS_%HH*z<+{&%2)X}`7`{> z`~lt<`j5~*h5jb=7opFEJ{kIGs2TdX(3#L|=y1pv{65Ku4+ww&2!H?xfB*=900@8p z2!O!r2BoS~LU-p5UGR#fEx6JU>{OtjCK4rwrCf5^ujg4<6FBfQ^*yl!@*KeM6xWR$_{M$1$2nW9z{^gUe&~#UV9Yms zY`7B(pzHtSkN)EW0w4eaAOHd&00JNY0w4eaAOHd&@IEF$*ZE=h}K>!3m00ck)1V8`;KmY_l00j030nGpRNJFEV zAOHd&00JNY0w4eaAOHd&00MiO0GY_w21~NzXUyMqXIXoDab;eSW2w3Lf-J}5ax9q?RV69MrBqx_$+D6X z(@8a@B+^P;Qj%gSAu5uXkQKEuf4^3ItP%F_x&Z zw0q6dZdEx^y?0s^)y&GdWuvxcTwcqi7Nwgh_0HRKnaibHYisw<$k&r^KF(gM%;oA! zx33v%OV>|S_0x&CbWV(kiKkv}$a9*t%C#!_ISJwHJdLLl1?OJi9{?diZMl$ zW1_tkNE4*D(+ZNfRV(D)K7Z%jot4|?YY#GaQt`)24>Q|pg)_J2-pD;^%%$}=&%d#J z^?%R!UZ^SR2m&Ag0w4eaAOHd& z00JNY0w4eadzS#_|9iLXQF9Og0T2KI5C8!X009sH0T2Lzy+Z)=|Gm@Js3{1500@8p z2!H?xfB*=900@A<-X(zf|K4qT)Eopr00ck)1V8`;KmY_l00cl_?-0QJfA6$4Y6=1% z00JNY0w4eaAOHd&00JPecL@XoKk)F}yB_|<(BBV#&Hu&VT)@%C%;Ri%gV{=JN zp-Qx+=#k{&&F&;VspU#~;l5TWR#mM~HY#eVT67+Nz^vEJ%$dts zA>vG?LiK(G{bWoOqg}7WNXs`#t<;s0R*VRZvQ{dQqJ`B; z!zfntLf29_dog?V5)s#9|1+lrF-n{e&y}LCmi0zME9xB;rgWNKCDo}k)#h4FXEM9h zqg7kZp3g34m(FHyba~h_m`I|+Icm4exGgR%-dxOFzI@j_nLS5J>GZ^Yc=i&y9c|lw z%_=vIGO1cYYwC2bOD5WSBOC}7dE;` zx9Qp0{hiq;0joILFnjePj^~wpI;OLx=)2o7ZFi)w|iy$GWuZfccYM^W94J%jt8>Y_~U>L`P3}u7#(jxOa~CBJ08!*#Oc8 zbXt!mhr+Gl47a{u^%w@ld9ioD(tECWRf=ngTtbdHzJ#5*51Fzy`Z{*nZ>Ka{M{`w7 ztIQHe`;y_kM>iR%CJqF`*U9aM;;4AL^H~>3I;kjWB_8i0Ht&%Rx?CQtxQ!e*^|tM( zO18(w&-JCDKscS|o+YjJZKqoD+ZRUP*!ivp5|H)>_A(ogMo0KeCOVEH2#MDP)jSiR*-Br!cNn}8}$dLVYhF0d^$UZ zGlRV&M_c0)f!%iJexE0Uh&RoxAGdJ!bv^5fvKoh)_CDe3A$XVt}dSIYGs~&3SVhGt>!Yr z==)x?GHml&y*A(v&(3mp8Q8|dlF`)FhW>C>ujF;dp|=O2Bk^QKE9<97&S>i{7YHvb zaO)cLY0j3)MzOBZX-lK;#6XYV&h*O4@LVmT)oQj0l%n)Zps3c3Q%Tm6p_7SO|GWX~ z|Bjdj4nY6}KmY_l00ck)1V8`;KmY_l;C)CS$W=VN=ZR(5A@-xaF{W;I zlJs2RrKZs=*+NK?=vbm(t(PpgeW^4ORv~Nm1wC(&<@@fsG@(t7>1upu>AyE?RGRmP zvr`-4w4$Fb~bQtUmey%ZJoe$2D~`Yf%6 zU-q-BMLy*ZUzpysV(p-DyJl>EYUI0~*mh_G=bhtEUnYB-E#(DVP|)D+bCIZN zjmG0@okSA5%^6Ge1R_gXqe-7QF)H?phdf|GdqP8PtdjoTEqqfY9+iLhwv z#RyqgpY2mEX{dp6*rV(3-Clh+Sy;@}1h0GSUKQ(2gZs^6fT47n!Uf?K}ENb4E z4usccxpm%Jh_SNt6_BqMbXsdwl(d|jR5WW1#>(Y4MY}BBT1MM+z3rNiM(Kzn&k4*3 z0lF~ojI)Lft7tYvSciiySB;hWT-S~h^XBW6<`;)OlUeE(YnApoTUV)LFz@Z8G5cad zv~_WcJV_*Ttq-x*cM+nJ^#rw~<@AzTCT(9*lZjkGF3367@GcpAu8r?vxVh(cySrqg zWV)K@Dm{}$I`ZNN0%2L^*3TT@o| z3BT@ga9&Bo^_-S)8XPsNWsVxoHit>mWZHF*+$?|3@Ms#+@!M8bB#QO_9gJ{0@l|L@X0w4eaAOHd&00JNY0w4eaAOHe8NC5Ny z9fS#2AOHd&00JNY0w4eaAOHd&00JPeO9=#tt^mL0;lIy+gZ~o$dH%Ef$H*Z*AOHd& z00JNY0w4eaAOHd&00JNY0y{xqBH;5JJ7WG>{)YMg5x>tfbKLy(lIB`XZ`k(727R8% zN%})Y^cPryK5`VMN9?D=_It_UG(EI_rfA=Q&l5hzevgIxB9~ri*4KE>=Q%QI{#YFS z**QA@_l)mEh2R4OKmY_l00ck)1V8`;KmY_l00cl_w-T^D|IhDMjnD-GAOHd&00JNY z0w4eaAOHd&00JQJlM%4Z|9>(^$PEG@00JNY0w4eaAOHd&00JNY0wAz!31I%eYnukb bKmY_l00ck)1V8`;KmY_l00cl_#|Zo%aCu#I literal 0 HcmV?d00001 diff --git a/scraper/templates/scraper/index.html b/scraper/templates/scraper/index.html index 9cb1a99..b28b722 100644 --- a/scraper/templates/scraper/index.html +++ b/scraper/templates/scraper/index.html @@ -22,17 +22,19 @@ + - - + + - {% for result in results %} + {% for item in latest_items %} - - - + + + + {% empty %} diff --git a/scraper/views.py b/scraper/views.py index 960e264..45782b8 100644 --- a/scraper/views.py +++ b/scraper/views.py @@ -5,58 +5,67 @@ from .utils import * from .scraper_class import FacebookScraper import re import statistics +from .models import Item class Index(View): def get(self, request): form = MarketForm() - return render(request, 'scraper/index.html', {'form': form}) + latest_items = Item.objects.all().order_by('-id')[:10] + context = {'form': form, 'latest_items': latest_items} + return render(request, 'scraper/index.html', context) def post(self, request): form = MarketForm(request.POST) if form.is_valid(): url = form.cleaned_data['url'] + shortened_url = re.search(r".*[0-9]", url).group(0) + mobile_url = shortened_url.replace("www", "m") + market_id = re.search(r"\/item\/([0-9]*)", url).group(1) + mobile_soup = create_soup(mobile_url, headers=None) + base_soup = create_soup(url, headers=None) + scraper_instance = FacebookScraper(mobile_soup, base_soup) - shortened_url = re.search(r".*[0-9]", url).group(0) - mobile_url = shortened_url.replace("www", "m") + listing_image = scraper_instance.get_listing_image() + listing_days, listing_hours = scraper_instance.get_listing_date() + listing_description = scraper_instance.get_listing_description() + title = scraper_instance.get_listing_title() + list_price = scraper_instance.get_listing_price() - market_id = re.search(r"\/item\/([0-9]*)", url).group(1) + sentiment_rating = sentiment_analysis(listing_description) - mobile_soup = create_soup(mobile_url, headers=None) - base_soup = create_soup(url, headers=None) - scraper_instance = FacebookScraper(mobile_soup, base_soup) + list_price = re.sub("[\$,]", "", list_price) + initial_price = int(re.sub("[\$,]", "", list_price)) - listing_image = scraper_instance.get_listing_image() - listing_days, listing_hours = scraper_instance.get_listing_date() - listing_description = scraper_instance.get_listing_description() - title = scraper_instance.get_listing_title() - list_price = scraper_instance.get_listing_price() + lower_bound, upper_bound, median = find_viable_product(title, ramp_down=0.0) + price_rating = price_difference_rating(initial_price, median) + average_rating = statistics.mean([sentiment_rating, price_rating]) - sentiment_rating = sentiment_analysis(listing_description) + # Create a new Item object + average_rating = round(average_rating, 1) + item = Item.objects.create(image=listing_image[0], title=title, rating=average_rating, url=shortened_url) - list_price = re.sub("[\$,]", "", list_price) - initial_price = int(re.sub("[\$,]", "", list_price)) + context = { + 'shortened_url': shortened_url, + 'mobile_url': mobile_url, + 'market_id': market_id, + 'sentiment_rating': round(sentiment_rating, 1), + 'title': title, + 'list_price': "{0:,.2f}".format(float(list_price)), + 'initial_price': initial_price, + 'lower_bound': "{0:,.2f}".format(lower_bound), + 'upper_bound': "{0:,.2f}".format(upper_bound), + 'median': "{0:,.2f}".format(median), + 'price_rating': round(price_rating, 1), + 'average_rating': average_rating, + 'days': listing_days, + 'hours': listing_hours, + 'image': listing_image[0], + 'id': market_id + } - lower_bound, upper_bound, median = find_viable_product(title, ramp_down=0.0) - price_rating = price_difference_rating(initial_price, median) - average_rating = statistics.mean([sentiment_rating, price_rating]) + return render(request, 'scraper/result.html', context) - context = { - 'shortened_url': shortened_url, - 'mobile_url': mobile_url, - 'market_id': market_id, - 'sentiment_rating': round(sentiment_rating, 1), - 'title': title, - 'list_price': "{0:,.2f}".format(float(list_price)), - 'initial_price': initial_price, - 'lower_bound': "{0:,.2f}".format(lower_bound), - 'upper_bound': "{0:,.2f}".format(upper_bound), - 'median': "{0:,.2f}".format(median), - 'price_rating': round(price_rating, 1), - 'average_rating': round(average_rating, 1), - 'days': listing_days, - 'hours': listing_hours, - 'image': listing_image[0], - 'id': market_id - } - - return render(request, 'scraper/result.html', context) \ No newline at end of file + else: + latest_items = Item.objects.all().order_by('-id')[:10] + context = {'form': form, 'latest_items': latest_items} + return render(request, 'scraper/index.html', context)
ItemScoreLinkRatingReport
{{ result.item }}{{ result.price }}{{ result.link }}{{ item.title }}{{ item.rating }}/5.0