diff --git a/crawler/2_dump_detail.py b/crawler/2_dump_detail.py index a7b0e18..dde735f 100644 --- a/crawler/2_dump_detail.py +++ b/crawler/2_dump_detail.py @@ -10,7 +10,13 @@ incremental = True listings = Listing.get_all_listings() filtered_listings = [] for listing in listings: - if not incremental and not listing.isRemoved: + # We introduced last_seen later, so not all entries have it. + # If it doesnt exist then its on the platform anymore. So skip + last_seen = listing.last_seen + if last_seen is None: + continue + + if not incremental and last_seen <= 1: filtered_listings.append(listing) if incremental and not listing.path_detail_json().exists(): diff --git a/crawler/data/decisions.json b/crawler/data/decisions.json index 5a3da17..a7e1e56 100644 --- a/crawler/data/decisions.json +++ b/crawler/data/decisions.json @@ -1 +1 @@ -{"identifier":{"351000":123629195.0,"1327000":130941260.0,"3990000":137017046.0,"6149000":139277669.0,"6518000":139590377.0,"6533000":139612598.0,"7943000":140724923.0,"8614000":141072356.0,"10086000":141881306.0,"10885000":142207478.0,"11527000":142497332.0,"12239000":142789514.0,"20339000":144509942.0,"25591000":145209752.0,"29287000":145680089.0,"29314000":145681742.0,"29631000":145712417.0,"30500000":145801379.0,"30617000":145815869.0,"31099000":145870028.0,"35066000":146250143.0,"37953000":146544002.0,"38329000":146570870.0,"42871000":146985392.0,"44592000":147163907.0,"44909000":147195593.0,"48902000":147585674.0,"49946000":147710042.0,"50039000":147723032.0,"52512000":148011845.0,"52796000":148033676.0,"58046000":148526264.0,"60308000":148884251.0,"60309000":148884416.0,"61481000":149130020.0,"61859000":149240837.0,"62028000":149291402.0,"62654000":149405942.0,"63680000":149587160.0,"63982000":149631728.0,"65029000":149811341.0,"66004000":149970839.0,"67597000":150253052.0,"68064000":150318137.0,"68466000":150380609.0,"68568000":150393980.0,"68710000":150412859.0,"68751000":150421832.0,"69137000":150489485.0,"72498000":150913952.0,"74158000":151085624.0,"74214000":151091798.0,"74314000":151101077.0,"75258000":151207043.0,"75952000":151519580.0,"75983000":151534727.0,"76066000":151562780.0,"77032000":151937687.0,"77123000":151965605.0,"77225000":151992926.0,"77230000":151993904.0,"77442000":152048477.0,"77646000":152100245.0,"77706000":152125295.0,"78179000":152261252.0,"78407000":152314865.0,"78595000":152365064.0,"78715000":152396093.0,"78935000":152489237.0,"78947000":152493332.0,"78977000":152497586.0,"80185000":152884436.0,"80440000":152948756.0,"80514000":152952956.0,"81811000":153011024.0,"82408000":153195794.0,"82484000":153226244.0,"82621000":153272789.0,"82641000":153281690.0,"82829000":153342827.0,"83494000":153541325.0,"83607000":153567278.0,"83709000":153593624.0,"83934000":153647309.0,"83980000":153659252.0,"84044000":153681545.0,"84397000":153806711.0,"84718000":153885416.0,"84885000":153931382.0,"86274000":154331711.0,"86335000":154344398.0,"86598000":154422317.0,"87192000":154597238.0,"87219000":154605446.0,"87250000":154612913.0,"87446000":154665839.0,"87577000":154695602.0,"87711000":154725011.0,"87734000":154728881.0,"87743000":154730432.0,"88732000":155006288.0,"88797000":155022773.0,"89262000":155154182.0,"89318000":155170724.0,"89319000":155170730.0,"89732000":155268164.0,"91569000":155902118.0,"97470000":86598888.0,"99359000":86914923.0,"99793000":87036150.0,"168.0":118752221.0,"1116.0":129764555.0,"1207.0":130397936.0,"1299.0":130849793.0,"9973.0":141826334.0,"12262.0":142793525.0,"14586.0":143408849.0,"17427.0":144018401.0,"26734.0":145386389.0,"34619.0":146214332.0,"39379.0":146660870.0,"50717.0":147806654.0,"51018.0":147842672.0,"58074.0":148528400.0,"58497.0":148564979.0,"61525.0":149137697.0,"66148.0":150019004.0,"67491.0":150237866.0,"68096.0":150321023.0,"69212.0":150497033.0,"70448.0":150652604.0,"71677.0":150816677.0,"73286.0":151006262.0,"74094.0":151074929.0,"75755.0":151413677.0,"76125.0":151576967.0,"76545.0":151755572.0,"77555.0":152069327.0,"77572.0":152073473.0,"78005.0":152207876.0,"78337.0":152290028.0,"78340.0":152290523.0,"78479.0":152328149.0,"79092.0":152516225.0,"79978.0":152793491.0,"80351.0":152929442.0,"80352.0":152929475.0,"80361.0":152929910.0,"80365.0":152930138.0,"80368.0":152930234.0,"80369.0":152930249.0,"80411.0":152940305.0,"80544.0":152952782.0,"80720.0":152961734.0,"81632.0":152988521.0,"81650.0":152988767.0,"81677.0":152989007.0,"81904.0":153020663.0,"82162.0":153095771.0,"82558.0":153234077.0,"83006.0":153385421.0,"83606.0":153554591.0,"83681.0":153572045.0,"84076.0":153677939.0,"84139.0":153706538.0,"84222.0":153733907.0,"84605.0":153841115.0,"84666.0":153856181.0,"84794.0":153890633.0,"85488.0":154068566.0,"86056.0":154229807.0,"86434.0":154356437.0,"86792.0":154455989.0,"87208.0":154582079.0,"87306.0":154612229.0,"87309.0":154612952.0,"87380.0":154631246.0,"87433.0":154650440.0,"87735.0":154718651.0,"87806.0":154730567.0,"88999.0":155061086.0,"89125.0":155103575.0,"89948.0":155324624.0,"89973.0":155334455.0,"90023.0":155349557.0,"90030.0":155351495.0,"90126.0":155387021.0,"90394.0":155510237.0,"91218.0":155802566.0,"91277.0":155814131.0,"91403.0":155851703.0,"91584.0":155877203.0,"91672.0":155908628.0,"91835.0":155968343.0,"91971.0":156021026.0,"92139.0":156072476.0,"92444.0":156170672.0,"92614.0":156195326.0,"92747.0":156202805.0,"93881.0":156464552.0,"93897.0":156478166.0,"94046.0":156538847.0,"94170.0":156578249.0,"94828.0":156788207.0,"94927.0":156808571.0,"94949.0":156814538.0,"95257.0":156885272.0,"95274.0":156889457.0,"95326.0":156896918.0,"95468.0":156925268.0,"95539.0":156953489.0,"95635.0":156976181.0,"95645.0":156977627.0,"96194.0":157133192.0,"96691.0":157248374.0,"97062.0":157316006.0,"97547.0":157453736.0,"97698.0":157483067.0,"97805.0":157504865.0,"97969.0":157540418.0,"98008.0":157549670.0,"98111.0":157576313.0,"98372.0":157633430.0,"100885.0":86868255.0},"decision":{"351000":"n, 82 years","1327000":"n","3990000":"n","6149000":"n, zu klein fuer den preis","6518000":"y, shortlist","6533000":"n, 52","7943000":"maybe","8614000":"n","10086000":"n, SC","10885000":"n, small","11527000":"n, nur 91 jahre lease","12239000":"n, lease expiring","20339000":"n, bungalow","25591000":"n, schmal","29287000":"y, maybe","29314000":"n","29631000":"maybe","30500000":"maybe","30617000":"y, maybe. Ground floor though","31099000":"n, alt","35066000":"y, love it","37953000":"n, schlecht erhalten","38329000":"n, kein licht","42871000":"melden","44592000":"n, small, expensive, far","44909000":"n","48902000":"y, weit aber maybe? Freehold!","49946000":"y, maybe","50039000":"n, zu klein","52512000":"n, too far from station","52796000":"n","58046000":"n, too narrow","60308000":"n, zu viel laufen","60309000":"n","61481000":"y, maybe","61859000":"n, am arsch der welt","62028000":"n, removed","62654000":"n","63680000":"n, 43 years","63982000":"n","65029000":"n, ground floor","66004000":"y","67597000":"n, removed","68064000":"maybe, altes listing","68466000":"revisit","68568000":"n, teuer + klein","68710000":"n","68751000":"n, boat","69137000":"n, nicht genug licht fuer den preis","72498000":"n, klein","74158000":"y, aber zu teuer","74214000":"n, ground floor","74314000":"n, zu teuer","75258000":"y aber billiger","75952000":"n, zu teuer","75983000":"n, lease fuer den preis?","76066000":"n","77032000":"n","77123000":"y","77225000":"y","77230000":"n, zu weit zu teuer","77442000":"n, ground floor","77646000":"n, schoen aber wohnzimmer zu klein","77706000":"n","78179000":"n","78407000":"y, maybe, dup of 152365064","78595000":"y, dup 152314865","78715000":"n","78935000":"n, ground floor","78947000":"gone","78977000":"gone","80185000":"j, vll","80440000":"n","80514000":"n, zu teuer","81811000":"n","82408000":"removed","82484000":"n, klein","82621000":"y","82641000":"n,","82829000":"n","83494000":"n, klein + high SC","83607000":"n","83709000":"n","83934000":"n, no lease + under offer","83980000":"n, sueden","84044000":"n, small","84397000":"n, erdgeschoss","84718000":"maybe","84885000":"n","86274000":"n, schrott","86335000":"n, too much work","86598000":"y","87192000":"n","87219000":"n, 36sqm","87250000":"n","87446000":"n, 46sqm","87577000":"n, zu duenn","87711000":"n, 50sqm","87734000":"n, zu duenn","87743000":"n, under offer","88732000":"gone","88797000":"revisit","89262000":"gone","89318000":"n, ground floor","89319000":"y, maybe","89732000":"n, gone","91569000":"n","97470000":"n","99359000":"n, weit, schlecht erhalten","99793000":"maybe","168.0":"n, teuer","1116.0":"n, nur land","1207.0":"n, nicht genug licht","1299.0":"j","9973.0":"n, SC","12262.0":"n, ugly","14586.0":"zu klein, der floorplan lueft","17427.0":"n","26734.0":"later","34619.0":"maybe","39379.0":"y, maybe","50717.0":"y, aber billiger?","51018.0":"n","58074.0":"n, schoen aber bad design","58497.0":"n, schmal","61525.0":"y, again same apartment","66148.0":"n, refurb needed","67491.0":"n, SC","68096.0":"n, meh","69212.0":"n, far","70448.0":"n, zu teuer","71677.0":"n, too far south","73286.0":"n, sc hoch","74094.0":"n, arbeit noetig","75755.0":"n, zu teuer","76125.0":"n, ground floor","76545.0":"y","77555.0":"n","77572.0":"n, schrecklich","78005.0":"y","78337.0":"y, same as 146970764","78340.0":"y, maybe","78479.0":"maybe","79092.0":"n, SC","79978.0":"n, nice aber wohnzimmer","80351.0":"n, 1b","80352.0":"vll, lease left zu kurz","80361.0":"n, SC","80365.0":"n","80368.0":"n, SC","80369.0":"n, SC","80411.0":"dup","80544.0":"n, south + too far from subway","80720.0":"n, small living room","81632.0":"maybe, anschauen","81650.0":"n, station far","81677.0":"y, strong","81904.0":"n, klein","82162.0":"n","82558.0":"n, erdgeschoss","83006.0":"y, maybe","83606.0":"n, small","83681.0":"n, too old style","84076.0":"n","84139.0":"y","84222.0":"n, ground floor","84605.0":"n, niemals","84666.0":"n","84794.0":"n, ground floor","85488.0":"n, small","86056.0":"n, SC 8000","86434.0":"n, SC vermutlich zu hoch","86792.0":"later","87208.0":"maybe","87306.0":"n, zu teuer 770k","87309.0":"n, dunkel fuer den preis","87380.0":"n","87433.0":"n, direkt an zug und stadium","87735.0":"y, maybe, long walk","87806.0":"y, aber nur fuer 730, und was ist die leasehold kost?","88999.0":"y, maybe","89125.0":"n, SC 9600","89948.0":"later","89973.0":"maybe? irgendwas ist kaputt hier","90023.0":"n","90030.0":"n, wrong qm","90126.0":"n, dup","90394.0":"n, eher nicht","91218.0":"n, ground floor","91277.0":"j, aber under offer","91403.0":"n","91584.0":"maybe","91672.0":"n","91835.0":"n, ground floor","91971.0":"y, aber zu teuer eigentlich","92139.0":"y, but under offer","92444.0":"n, 5800 SC","92614.0":"n, under offer + zu weit von next transport","92747.0":"n, 1 br","93881.0":"y","93897.0":"n, SC 5000","94046.0":"y, maybe","94170.0":"dup","94828.0":"n, nicht in london","94927.0":"n, plus under offer anyway","94949.0":"n, plus under offer anyway + dup","95257.0":"y, but sold stc","95274.0":"n, schlecht geschnitten, zu teuer","95326.0":"y","95468.0":"m, gut aber zu teuer","95539.0":"m, SC anfragen","95635.0":"n, uberzeugt mich irgendwie nicht","95645.0":"n","96194.0":"n, kleines wohnzimmer","96691.0":"n","97062.0":"n, zuviel arbeit","97547.0":"later","97698.0":"n, kein licht","97805.0":"n, sueden","97969.0":"y, 8min laufweg","98008.0":"n, zu weit","98111.0":"n","98372.0":"n, falsches OCR","100885.0":"n"}} \ No newline at end of file +{"identifier":{"351000000":123629195.0,"1327000000":130941260.0,"3990000000":137017046.0,"6149000000":139277669.0,"6518000000":139590377.0,"6533000000":139612598.0,"7943000000":140724923.0,"8614000000":141072356.0,"10086000000":141881306.0,"10885000000":142207478.0,"11527000000":142497332.0,"12239000000":142789514.0,"20339000000":144509942.0,"25591000000":145209752.0,"29287000000":145680089.0,"29314000000":145681742.0,"29631000000":145712417.0,"30500000000":145801379.0,"30617000000":145815869.0,"31099000000":145870028.0,"35066000000":146250143.0,"37953000000":146544002.0,"38329000000":146570870.0,"42871000000":146985392.0,"44592000000":147163907.0,"44909000000":147195593.0,"48902000000":147585674.0,"49946000000":147710042.0,"50039000000":147723032.0,"52512000000":148011845.0,"52796000000":148033676.0,"58046000000":148526264.0,"60308000000":148884251.0,"60309000000":148884416.0,"61481000000":149130020.0,"61859000000":149240837.0,"62028000000":149291402.0,"62654000000":149405942.0,"63680000000":149587160.0,"63982000000":149631728.0,"65029000000":149811341.0,"66004000000":149970839.0,"67597000000":150253052.0,"68064000000":150318137.0,"68466000000":150380609.0,"68568000000":150393980.0,"68710000000":150412859.0,"68751000000":150421832.0,"69137000000":150489485.0,"72498000000":150913952.0,"74158000000":151085624.0,"74214000000":151091798.0,"74314000000":151101077.0,"75258000000":151207043.0,"75952000000":151519580.0,"75983000000":151534727.0,"76066000000":151562780.0,"77032000000":151937687.0,"77123000000":151965605.0,"77225000000":151992926.0,"77230000000":151993904.0,"77442000000":152048477.0,"77646000000":152100245.0,"77706000000":152125295.0,"78179000000":152261252.0,"78407000000":152314865.0,"78595000000":152365064.0,"78715000000":152396093.0,"78935000000":152489237.0,"78947000000":152493332.0,"78977000000":152497586.0,"80185000000":152884436.0,"80440000000":152948756.0,"80514000000":152952956.0,"81811000000":153011024.0,"82408000000":153195794.0,"82484000000":153226244.0,"82621000000":153272789.0,"82641000000":153281690.0,"82829000000":153342827.0,"83494000000":153541325.0,"83607000000":153567278.0,"83709000000":153593624.0,"83934000000":153647309.0,"83980000000":153659252.0,"84044000000":153681545.0,"84397000000":153806711.0,"84718000000":153885416.0,"84885000000":153931382.0,"86274000000":154331711.0,"86335000000":154344398.0,"86598000000":154422317.0,"87192000000":154597238.0,"87219000000":154605446.0,"87250000000":154612913.0,"87446000000":154665839.0,"87577000000":154695602.0,"87711000000":154725011.0,"87734000000":154728881.0,"87743000000":154730432.0,"88732000000":155006288.0,"88797000000":155022773.0,"89262000000":155154182.0,"89318000000":155170724.0,"89319000000":155170730.0,"89732000000":155268164.0,"91569000000":155902118.0,"97470000000":86598888.0,"99359000000":86914923.0,"99793000000":87036150.0,"168.0":118752221.0,"1116.0":129764555.0,"12266.0":142793525.0,"17432.0":144018401.0,"34626.0":146214332.0,"66162.0":150019004.0,"81936.0":153020663.0,"83641.0":153554591.0,"85525.0":154068566.0,"87419.0":154631246.0,"90071.0":155351495.0,"95738.0":156977627.0,"51028.0":147842672.0,"74112.0":151074929.0,"82593.0":153234077.0,"87246.0":154582079.0,"97168.0":157316006.0,"76146.0":151576967.0,"91330.0":155814131.0,"58084.0":148528400.0,"58507.0":148564979.0,"75774.0":151413677.0,"69228.0":150497033.0,"81663.0":152988521.0,"87396.0":154626839.0,"1207.0":130397936.0,"102352.0":86868255.0,"78361.0":152290028.0,"96793.0":157248374.0,"81681.0":152988767.0,"78028.0":152207876.0,"83041.0":153385421.0,"39386.0":146660870.0,"77578.0":152069327.0,"84701.0":153856181.0,"90014.0":155334455.0,"91725.0":155908628.0,"84111.0":153677939.0,"9976.0":141826334.0,"26740.0":145386389.0,"99724.0":157936007.0,"91456.0":155851703.0,"83716.0":153572045.0,"89988.0":155324624.0,"95344.0":156885272.0,"84174.0":153706538.0,"97915.0":157504865.0,"94239.0":156578249.0,"92197.0":156072476.0,"87774.0":154718651.0,"91271.0":155802566.0,"84257.0":153733907.0,"95414.0":156896918.0,"95561.0":156925268.0,"80437.0":152940305.0,"92504.0":156170672.0,"68112.0":150321023.0,"98710.0":157690190.0,"77595.0":152073473.0,"70465.0":150652604.0,"84830.0":153890633.0,"92027.0":156021026.0,"95632.0":156953489.0,"95008.0":156808571.0,"95031.0":156814538.0,"95728.0":156976181.0,"98090.0":157540418.0,"87845.0":154730567.0,"92674.0":156195326.0,"73304.0":151006262.0,"86093.0":154229807.0,"82196.0":153095771.0,"90439.0":155510237.0,"67507.0":150237866.0,"98129.0":157549670.0,"86829.0":154455989.0,"95361.0":156889457.0,"97655.0":157453736.0,"79116.0":152516225.0,"98232.0":157576313.0,"80377.0":152929475.0,"97651.0":157452611.0,"77853.0":152146979.0,"80376.0":152929442.0,"81709.0":152989007.0,"99818.0":157952573.0,"91888.0":155968343.0,"94113.0":156538847.0,"80003.0":152793491.0,"1299.0":130849793.0,"99527.0":157882673.0,"98495.0":157633430.0,"86471.0":154356437.0,"90167.0":155387021.0,"78364.0":152290523.0,"88222.0":154849886.0,"96294.0":157133192.0,"78503.0":152328149.0,"50727.0":147806654.0,"99960.0":157988078.0,"97808.0":157483067.0,"99858.0":157959716.0,"61537.0":149137697.0,"80571.0":152952782.0,"80395.0":152930249.0,"87347.0":154612952.0,"98964.0":157741886.0,"94528.0":156666197.0,"80391.0":152930138.0,"99434.0":157866818.0,"98612.0":157647914.0,"98819.0":157711766.0,"14591.0":143408849.0,"98990.0":157746227.0,"92808.0":156202805.0,"98654.0":157663343.0,"84640.0":153841115.0,"71694.0":150816677.0,"76567.0":151755572.0,"97548.0":157430714.0,"89039.0":155061086.0,"94109.0":156538406.0,"80394.0":152930234.0,"93963.0":156478166.0,"80386.0":152929910.0,"87472.0":154650440.0,"89165.0":155103575.0,"93945.0":156464552.0,"90064.0":155349557.0,"80747.0":152961734.0,"91637.0":155877203.0,"87344.0":154612229.0,"94908.0":156788207.0},"decision":{"351000000":"n, 82 years","1327000000":"n","3990000000":"n","6149000000":"n, zu klein fuer den preis","6518000000":"y, shortlist","6533000000":"n, 52","7943000000":"maybe","8614000000":"n","10086000000":"n, SC","10885000000":"n, small","11527000000":"n, nur 91 jahre lease","12239000000":"n, lease expiring","20339000000":"n, bungalow","25591000000":"n, schmal","29287000000":"y, maybe","29314000000":"n","29631000000":"maybe","30500000000":"maybe","30617000000":"y, maybe. Ground floor though","31099000000":"n, alt","35066000000":"y, love it","37953000000":"n, schlecht erhalten","38329000000":"n, kein licht","42871000000":"melden","44592000000":"n, small, expensive, far","44909000000":"n","48902000000":"y, weit aber maybe? Freehold!","49946000000":"y, maybe","50039000000":"n, zu klein","52512000000":"n, too far from station","52796000000":"n","58046000000":"n, too narrow","60308000000":"n, zu viel laufen","60309000000":"n","61481000000":"y, maybe","61859000000":"n, am arsch der welt","62028000000":"n, removed","62654000000":"n","63680000000":"n, 43 years","63982000000":"n","65029000000":"n, ground floor","66004000000":"y","67597000000":"n, removed","68064000000":"maybe, altes listing","68466000000":"revisit","68568000000":"n, teuer + klein","68710000000":"n","68751000000":"n, boat","69137000000":"n, nicht genug licht fuer den preis","72498000000":"n, klein","74158000000":"y, aber zu teuer","74214000000":"n, ground floor","74314000000":"n, zu teuer","75258000000":"y aber billiger","75952000000":"n, zu teuer","75983000000":"n, lease fuer den preis?","76066000000":"n","77032000000":"n","77123000000":"y","77225000000":"y","77230000000":"n, zu weit zu teuer","77442000000":"n, ground floor","77646000000":"n, schoen aber wohnzimmer zu klein","77706000000":"n","78179000000":"n","78407000000":"y, maybe, dup of 152365064","78595000000":"y, dup 152314865","78715000000":"n","78935000000":"n, ground floor","78947000000":"gone","78977000000":"gone","80185000000":"j, vll","80440000000":"n","80514000000":"n, zu teuer","81811000000":"n","82408000000":"removed","82484000000":"n, klein","82621000000":"y","82641000000":"n,","82829000000":"n","83494000000":"n, klein + high SC","83607000000":"n","83709000000":"n","83934000000":"n, no lease + under offer","83980000000":"n, sueden","84044000000":"n, small","84397000000":"n, erdgeschoss","84718000000":"maybe","84885000000":"n","86274000000":"n, schrott","86335000000":"n, too much work","86598000000":"y","87192000000":"n","87219000000":"n, 36sqm","87250000000":"n","87446000000":"n, 46sqm","87577000000":"n, zu duenn","87711000000":"n, 50sqm","87734000000":"n, zu duenn","87743000000":"n, under offer","88732000000":"gone","88797000000":"revisit","89262000000":"gone","89318000000":"n, ground floor","89319000000":"y, maybe","89732000000":"n, gone","91569000000":"n","97470000000":"n","99359000000":"n, weit, schlecht erhalten","99793000000":"maybe","168.0":"n, teuer","1116.0":"n, nur land","12266.0":"n, ugly","17432.0":"n","34626.0":"maybe","66162.0":"n, refurb needed","81936.0":"n, klein","83641.0":"n, small","85525.0":"n, small","87419.0":"n","90071.0":"n, wrong qm","95738.0":"n","51028.0":"n","74112.0":"n, arbeit noetig","82593.0":"n, erdgeschoss","87246.0":"maybe","97168.0":"n, zuviel arbeit","76146.0":"n, ground floor","91330.0":"j, aber under offer","58084.0":"n, schoen aber bad design","58507.0":"n, schmal","75774.0":"n, zu teuer","69228.0":"n, far","81663.0":"maybe, anschauen","87396.0":"n, 1br","1207.0":"n, nicht genug licht","102352.0":"n","78361.0":"y, same as 146970764","96793.0":"n","81681.0":"n, station far","78028.0":"y","83041.0":"y, maybe","39386.0":"y, maybe","77578.0":"n","84701.0":"n","90014.0":"maybe? irgendwas ist kaputt hier","91725.0":"n","84111.0":"n","9976.0":"n, SC","26740.0":"later","99724.0":"n, am stadium","91456.0":"n","83716.0":"n, too old style","89988.0":"later","95344.0":"y, but sold stc","84174.0":"y","97915.0":"n, sueden","94239.0":"dup","92197.0":"y, but under offer","87774.0":"y, maybe, long walk","91271.0":"n, ground floor","84257.0":"n, ground floor","95414.0":"y","95561.0":"m, gut aber zu teuer","80437.0":"dup","92504.0":"n, 5800 SC","68112.0":"n, meh","98710.0":"n, south","77595.0":"n, schrecklich","70465.0":"n, zu teuer","84830.0":"n, ground floor","92027.0":"y, aber zu teuer eigentlich","95632.0":"m, SC anfragen","95008.0":"n, plus under offer anyway","95031.0":"n, plus under offer anyway + dup","95728.0":"n, uberzeugt mich irgendwie nicht","98090.0":"y, 8min laufweg","87845.0":"y, aber nur fuer 730, und was ist die leasehold kost?","92674.0":"n, under offer + zu weit von next transport","73304.0":"n, sc hoch","86093.0":"n, SC 8000","82196.0":"n","90439.0":"n, eher nicht","67507.0":"n, SC","98129.0":"n, zu weit","86829.0":"later","95361.0":"n, schlecht geschnitten, zu teuer","97655.0":"later","79116.0":"n, SC","98232.0":"n","80377.0":"vll, lease left zu kurz","97651.0":"y, maybe","77853.0":"n, sueden","80376.0":"n, 1b","81709.0":"y, strong","99818.0":"y, maybe, expensive, a bit far out but really nice","91888.0":"n, ground floor","94113.0":"y, maybe","80003.0":"n, nice aber wohnzimmer","1299.0":"j","99527.0":"n, zu teuer fuer suden","98495.0":"n, falsches OCR","86471.0":"n, SC vermutlich zu hoch","90167.0":"n, dup","78364.0":"y, maybe","88222.0":"n, weit weg","96294.0":"n, kleines wohnzimmer","78503.0":"maybe","50727.0":"y, aber billiger?","99960.0":"n, sueden","97808.0":"n, kein licht","99858.0":"n, gibt bessere fuer den preis","61537.0":"y, again same apartment","80571.0":"n, south + too far from subway","80395.0":"n, SC","87347.0":"n, dunkel fuer den preis","98964.0":"n, gibt bessere","94528.0":"n, irgendwie nicht so","80391.0":"n","99434.0":"n, zuviel SC und zuwenig licht fuer den preis","98612.0":"y, maybe, second look","98819.0":"n, gibt bessere","14591.0":"zu klein, der floorplan lueft","98990.0":"n, schlechtes licht","92808.0":"n, 1 br","98654.0":"n, 1 br fuer den preis zu teuer","84640.0":"n, niemals","71694.0":"n, too far south","76567.0":"y","97548.0":"n, wtf","89039.0":"y, maybe","94109.0":"n, kein licht","80394.0":"n, SC","93963.0":"n, SC 5000","80386.0":"n, SC","87472.0":"n, direkt an zug und stadium","89165.0":"n, SC 9600","93945.0":"y","90064.0":"n","80747.0":"n, small living room","91637.0":"maybe","87344.0":"n, zu teuer 770k","94908.0":"n, nicht in london"}} \ No newline at end of file diff --git a/crawler/data_access.py b/crawler/data_access.py index e894f9b..cb6cbef 100644 --- a/crawler/data_access.py +++ b/crawler/data_access.py @@ -163,6 +163,12 @@ class Listing: def url(self): return f"https://www.rightmove.co.uk/properties/{self.identifier}" + @property + def listingobject(self): + if self._cached is None: + with open(self.path_listing_json()) as f: + return json.load(f) + @property def detailobject(self): if self._cached is None: @@ -214,7 +220,7 @@ class Listing: return (now - ds).days @property - def last_seen(self) -> datetime.datetime: + def last_seen(self) -> int: if not self.path_last_seen_listing().exists(): return None @@ -246,7 +252,7 @@ class Listing: @property def isRemoved(self) -> bool: - return not self.detailobject["property"]["visible"] + return not self.listingobject["visible"] @property def status(self) -> str: