Modern ActivityPub compliant server, designed for simplicity and accessibility. Includes calendar and sharing economy features to empower your federated community. https://code.freedombone.net/bashrc/epicyon Docs: https://epicyon.net/#install
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

inbox.py 129KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230
  1. __filename__ = "inbox.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.2.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import json
  9. import os
  10. import datetime
  11. import time
  12. from linked_data_sig import verifyJsonSignature
  13. from utils import dmAllowedFromDomain
  14. from utils import isRecentPost
  15. from utils import getConfigParam
  16. from utils import hasUsersPath
  17. from utils import validPostDate
  18. from utils import getFullDomain
  19. from utils import isEventPost
  20. from utils import removeIdEnding
  21. from utils import getProtocolPrefixes
  22. from utils import isBlogPost
  23. from utils import removeAvatarFromCache
  24. from utils import isPublicPost
  25. from utils import getCachedPostFilename
  26. from utils import removePostFromCache
  27. from utils import urlPermitted
  28. from utils import createInboxQueueDir
  29. from utils import getStatusNumber
  30. from utils import getDomainFromActor
  31. from utils import getNicknameFromActor
  32. from utils import locatePost
  33. from utils import deletePost
  34. from utils import removeModerationPostFromIndex
  35. from utils import loadJson
  36. from utils import saveJson
  37. from utils import updateLikesCollection
  38. from utils import undoLikesCollectionEntry
  39. from categories import getHashtagCategories
  40. from categories import setHashtagCategory
  41. from httpsig import verifyPostHeaders
  42. from session import createSession
  43. from session import getJson
  44. from follow import isFollowingActor
  45. from follow import receiveFollowRequest
  46. from follow import getFollowersOfActor
  47. from follow import unfollowerOfAccount
  48. from pprint import pprint
  49. from cache import getPersonFromCache
  50. from cache import storePersonInCache
  51. from acceptreject import receiveAcceptReject
  52. from bookmarks import updateBookmarksCollection
  53. from bookmarks import undoBookmarksCollectionEntry
  54. from blocking import isBlocked
  55. from blocking import isBlockedDomain
  56. from blocking import brochModeLapses
  57. from filters import isFiltered
  58. from utils import updateAnnounceCollection
  59. from utils import undoAnnounceCollectionEntry
  60. from utils import dangerousMarkup
  61. from utils import isDM
  62. from utils import isReply
  63. from httpsig import messageContentDigest
  64. from posts import createDirectMessagePost
  65. from posts import validContentWarning
  66. from posts import downloadAnnounce
  67. from posts import isMuted
  68. from posts import isImageMedia
  69. from posts import sendSignedJson
  70. from posts import sendToFollowersThread
  71. from webapp_post import individualPostAsHtml
  72. from question import questionUpdateVotes
  73. from media import replaceYouTube
  74. from git import isGitPatch
  75. from git import receiveGitPatch
  76. from followingCalendar import receivingCalendarEvents
  77. from happening import saveEventPost
  78. from delete import removeOldHashtags
  79. from categories import guessHashtagCategory
  80. from context import hasValidContext
  81. from speaker import updateSpeaker
  82. def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
  83. """Extracts hashtags from an incoming post and updates the
  84. relevant tags files.
  85. """
  86. if not isPublicPost(postJsonObject):
  87. return
  88. if not postJsonObject.get('object'):
  89. return
  90. if not isinstance(postJsonObject['object'], dict):
  91. return
  92. if not postJsonObject['object'].get('tag'):
  93. return
  94. if not postJsonObject.get('id'):
  95. return
  96. if not isinstance(postJsonObject['object']['tag'], list):
  97. return
  98. tagsDir = baseDir + '/tags'
  99. # add tags directory if it doesn't exist
  100. if not os.path.isdir(tagsDir):
  101. print('Creating tags directory')
  102. os.mkdir(tagsDir)
  103. hashtagCategories = getHashtagCategories(baseDir)
  104. for tag in postJsonObject['object']['tag']:
  105. if not tag.get('type'):
  106. continue
  107. if not isinstance(tag['type'], str):
  108. continue
  109. if tag['type'] != 'Hashtag':
  110. continue
  111. if not tag.get('name'):
  112. continue
  113. tagName = tag['name'].replace('#', '').strip()
  114. tagsFilename = tagsDir + '/' + tagName + '.txt'
  115. postUrl = removeIdEnding(postJsonObject['id'])
  116. postUrl = postUrl.replace('/', '#')
  117. daysDiff = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
  118. daysSinceEpoch = daysDiff.days
  119. tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n'
  120. if not os.path.isfile(tagsFilename):
  121. tagsFile = open(tagsFilename, "w+")
  122. if tagsFile:
  123. tagsFile.write(tagline)
  124. tagsFile.close()
  125. else:
  126. if postUrl not in open(tagsFilename).read():
  127. try:
  128. with open(tagsFilename, 'r+') as tagsFile:
  129. content = tagsFile.read()
  130. if tagline not in content:
  131. tagsFile.seek(0, 0)
  132. tagsFile.write(tagline + content)
  133. except Exception as e:
  134. print('WARN: Failed to write entry to tags file ' +
  135. tagsFilename + ' ' + str(e))
  136. removeOldHashtags(baseDir, 3)
  137. # automatically assign a category to the tag if possible
  138. categoryFilename = tagsDir + '/' + tagName + '.category'
  139. if not os.path.isfile(categoryFilename):
  140. categoryStr = \
  141. guessHashtagCategory(tagName, hashtagCategories)
  142. if categoryStr:
  143. setHashtagCategory(baseDir, tagName, categoryStr)
  144. def _inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
  145. translate: {},
  146. baseDir: str, httpPrefix: str,
  147. session, cachedWebfingers: {}, personCache: {},
  148. nickname: str, domain: str, port: int,
  149. postJsonObject: {},
  150. allowDeletion: bool, boxname: str,
  151. showPublishedDateOnly: bool,
  152. peertubeInstances: [],
  153. allowLocalNetworkAccess: bool,
  154. themeName: str) -> None:
  155. """Converts the json post into html and stores it in a cache
  156. This enables the post to be quickly displayed later
  157. """
  158. pageNumber = -999
  159. avatarUrl = None
  160. if boxname != 'tlevents' and boxname != 'outbox':
  161. boxname = 'inbox'
  162. individualPostAsHtml(True, recentPostsCache, maxRecentPosts,
  163. translate, pageNumber,
  164. baseDir, session, cachedWebfingers,
  165. personCache,
  166. nickname, domain, port, postJsonObject,
  167. avatarUrl, True, allowDeletion,
  168. httpPrefix, __version__, boxname, None,
  169. showPublishedDateOnly,
  170. peertubeInstances, allowLocalNetworkAccess,
  171. themeName,
  172. not isDM(postJsonObject),
  173. True, True, False, True)
  174. def validInbox(baseDir: str, nickname: str, domain: str) -> bool:
  175. """Checks whether files were correctly saved to the inbox
  176. """
  177. if ':' in domain:
  178. domain = domain.split(':')[0]
  179. inboxDir = baseDir+'/accounts/' + nickname + '@' + domain + '/inbox'
  180. if not os.path.isdir(inboxDir):
  181. return True
  182. for subdir, dirs, files in os.walk(inboxDir):
  183. for f in files:
  184. filename = os.path.join(subdir, f)
  185. if not os.path.isfile(filename):
  186. print('filename: ' + filename)
  187. return False
  188. if 'postNickname' in open(filename).read():
  189. print('queue file incorrectly saved to ' + filename)
  190. return False
  191. break
  192. return True
  193. def validInboxFilenames(baseDir: str, nickname: str, domain: str,
  194. expectedDomain: str, expectedPort: int) -> bool:
  195. """Used by unit tests to check that the port number gets appended to
  196. domain names within saved post filenames
  197. """
  198. if ':' in domain:
  199. domain = domain.split(':')[0]
  200. inboxDir = baseDir + '/accounts/' + nickname + '@' + domain + '/inbox'
  201. if not os.path.isdir(inboxDir):
  202. return True
  203. expectedStr = expectedDomain + ':' + str(expectedPort)
  204. for subdir, dirs, files in os.walk(inboxDir):
  205. for f in files:
  206. filename = os.path.join(subdir, f)
  207. if not os.path.isfile(filename):
  208. print('filename: ' + filename)
  209. return False
  210. if expectedStr not in filename:
  211. print('Expected: ' + expectedStr)
  212. print('Invalid filename: ' + filename)
  213. return False
  214. break
  215. return True
  216. def getPersonPubKey(baseDir: str, session, personUrl: str,
  217. personCache: {}, debug: bool,
  218. projectVersion: str, httpPrefix: str,
  219. domain: str, onionDomain: str) -> str:
  220. if not personUrl:
  221. return None
  222. personUrl = personUrl.replace('#main-key', '')
  223. if personUrl.endswith('/users/inbox'):
  224. if debug:
  225. print('DEBUG: Obtaining public key for shared inbox')
  226. personUrl = personUrl.replace('/users/inbox', '/inbox')
  227. personJson = \
  228. getPersonFromCache(baseDir, personUrl, personCache, True)
  229. if not personJson:
  230. if debug:
  231. print('DEBUG: Obtaining public key for ' + personUrl)
  232. personDomain = domain
  233. if onionDomain:
  234. if '.onion/' in personUrl:
  235. personDomain = onionDomain
  236. profileStr = 'https://www.w3.org/ns/activitystreams'
  237. asHeader = {
  238. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  239. }
  240. personJson = \
  241. getJson(session, personUrl, asHeader, None, debug,
  242. projectVersion, httpPrefix, personDomain)
  243. if not personJson:
  244. return None
  245. pubKey = None
  246. if personJson.get('publicKey'):
  247. if personJson['publicKey'].get('publicKeyPem'):
  248. pubKey = personJson['publicKey']['publicKeyPem']
  249. else:
  250. if personJson.get('publicKeyPem'):
  251. pubKey = personJson['publicKeyPem']
  252. if not pubKey:
  253. if debug:
  254. print('DEBUG: Public key not found for ' + personUrl)
  255. storePersonInCache(baseDir, personUrl, personJson, personCache, True)
  256. return pubKey
  257. def inboxMessageHasParams(messageJson: {}) -> bool:
  258. """Checks whether an incoming message contains expected parameters
  259. """
  260. expectedParams = ['actor', 'type', 'object']
  261. for param in expectedParams:
  262. if not messageJson.get(param):
  263. # print('inboxMessageHasParams: ' +
  264. # param + ' ' + str(messageJson))
  265. return False
  266. # actor should be a string
  267. if not isinstance(messageJson['actor'], str):
  268. print('WARN: actor should be a string, but is actually: ' +
  269. str(messageJson['actor']))
  270. return False
  271. # type should be a string
  272. if not isinstance(messageJson['type'], str):
  273. print('WARN: type from ' + str(messageJson['actor']) +
  274. ' should be a string, but is actually: ' +
  275. str(messageJson['type']))
  276. return False
  277. # object should be a dict or a string
  278. if not isinstance(messageJson['object'], dict):
  279. if not isinstance(messageJson['object'], str):
  280. print('WARN: object from ' + str(messageJson['actor']) +
  281. ' should be a dict or string, but is actually: ' +
  282. str(messageJson['object']))
  283. return False
  284. if not messageJson.get('to'):
  285. allowedWithoutToParam = ['Like', 'Follow', 'Join', 'Request',
  286. 'Accept', 'Capability', 'Undo']
  287. if messageJson['type'] not in allowedWithoutToParam:
  288. return False
  289. return True
  290. def inboxPermittedMessage(domain: str, messageJson: {},
  291. federationList: []) -> bool:
  292. """ check that we are receiving from a permitted domain
  293. """
  294. if not messageJson.get('actor'):
  295. return False
  296. actor = messageJson['actor']
  297. # always allow the local domain
  298. if domain in actor:
  299. return True
  300. if not urlPermitted(actor, federationList):
  301. return False
  302. alwaysAllowedTypes = ('Follow', 'Join', 'Like', 'Delete', 'Announce')
  303. if messageJson['type'] not in alwaysAllowedTypes:
  304. if not messageJson.get('object'):
  305. return True
  306. if not isinstance(messageJson['object'], dict):
  307. return False
  308. if messageJson['object'].get('inReplyTo'):
  309. inReplyTo = messageJson['object']['inReplyTo']
  310. if not isinstance(inReplyTo, str):
  311. return False
  312. if not urlPermitted(inReplyTo, federationList):
  313. return False
  314. return True
  315. def savePostToInboxQueue(baseDir: str, httpPrefix: str,
  316. nickname: str, domain: str,
  317. postJsonObject: {},
  318. originalPostJsonObject: {},
  319. messageBytes: str,
  320. httpHeaders: {},
  321. postPath: str, debug: bool) -> str:
  322. """Saves the give json to the inbox queue for the person
  323. keyId specifies the actor sending the post
  324. """
  325. if len(messageBytes) > 10240:
  326. print('WARN: inbox message too long ' +
  327. str(len(messageBytes)) + ' bytes')
  328. return None
  329. originalDomain = domain
  330. if ':' in domain:
  331. domain = domain.split(':')[0]
  332. # block at the ealiest stage possible, which means the data
  333. # isn't written to file
  334. postNickname = None
  335. postDomain = None
  336. actor = None
  337. if postJsonObject.get('actor'):
  338. if not isinstance(postJsonObject['actor'], str):
  339. return None
  340. actor = postJsonObject['actor']
  341. postNickname = getNicknameFromActor(postJsonObject['actor'])
  342. if not postNickname:
  343. print('No post Nickname in actor ' + postJsonObject['actor'])
  344. return None
  345. postDomain, postPort = getDomainFromActor(postJsonObject['actor'])
  346. if not postDomain:
  347. if debug:
  348. pprint(postJsonObject)
  349. print('No post Domain in actor')
  350. return None
  351. if isBlocked(baseDir, nickname, domain, postNickname, postDomain):
  352. if debug:
  353. print('DEBUG: post from ' + postNickname + ' blocked')
  354. return None
  355. postDomain = getFullDomain(postDomain, postPort)
  356. if postJsonObject.get('object'):
  357. if isinstance(postJsonObject['object'], dict):
  358. if postJsonObject['object'].get('inReplyTo'):
  359. if isinstance(postJsonObject['object']['inReplyTo'], str):
  360. inReplyTo = \
  361. postJsonObject['object']['inReplyTo']
  362. replyDomain, replyPort = \
  363. getDomainFromActor(inReplyTo)
  364. if isBlockedDomain(baseDir, replyDomain):
  365. if debug:
  366. print('WARN: post contains reply from ' +
  367. str(actor) +
  368. ' to a blocked domain: ' + replyDomain)
  369. return None
  370. else:
  371. replyNickname = \
  372. getNicknameFromActor(inReplyTo)
  373. if replyNickname and replyDomain:
  374. if isBlocked(baseDir, nickname, domain,
  375. replyNickname, replyDomain):
  376. if debug:
  377. print('WARN: post contains reply from ' +
  378. str(actor) +
  379. ' to a blocked account: ' +
  380. replyNickname + '@' + replyDomain)
  381. return None
  382. if postJsonObject['object'].get('content'):
  383. if isinstance(postJsonObject['object']['content'], str):
  384. if isFiltered(baseDir, nickname, domain,
  385. postJsonObject['object']['content']):
  386. if debug:
  387. print('WARN: post was filtered out due to content')
  388. return None
  389. originalPostId = None
  390. if postJsonObject.get('id'):
  391. if not isinstance(postJsonObject['id'], str):
  392. return None
  393. originalPostId = removeIdEnding(postJsonObject['id'])
  394. currTime = datetime.datetime.utcnow()
  395. postId = None
  396. if postJsonObject.get('id'):
  397. postId = removeIdEnding(postJsonObject['id'])
  398. published = currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  399. if not postId:
  400. statusNumber, published = getStatusNumber()
  401. if actor:
  402. postId = actor + '/statuses/' + statusNumber
  403. else:
  404. postId = httpPrefix + '://' + originalDomain + \
  405. '/users/' + nickname + '/statuses/' + statusNumber
  406. # NOTE: don't change postJsonObject['id'] before signature check
  407. inboxQueueDir = createInboxQueueDir(nickname, domain, baseDir)
  408. handle = nickname + '@' + domain
  409. destination = baseDir + '/accounts/' + \
  410. handle + '/inbox/' + postId.replace('/', '#') + '.json'
  411. filename = inboxQueueDir + '/' + postId.replace('/', '#') + '.json'
  412. sharedInboxItem = False
  413. if nickname == 'inbox':
  414. nickname = originalDomain
  415. sharedInboxItem = True
  416. digestStartTime = time.time()
  417. digest = messageContentDigest(messageBytes)
  418. timeDiffStr = str(int((time.time() - digestStartTime) * 1000))
  419. if debug:
  420. while len(timeDiffStr) < 6:
  421. timeDiffStr = '0' + timeDiffStr
  422. print('DIGEST|' + timeDiffStr + '|' + filename)
  423. newQueueItem = {
  424. 'originalId': originalPostId,
  425. 'id': postId,
  426. 'actor': actor,
  427. 'nickname': nickname,
  428. 'domain': domain,
  429. 'postNickname': postNickname,
  430. 'postDomain': postDomain,
  431. 'sharedInbox': sharedInboxItem,
  432. 'published': published,
  433. 'httpHeaders': httpHeaders,
  434. 'path': postPath,
  435. 'post': postJsonObject,
  436. 'original': originalPostJsonObject,
  437. 'digest': digest,
  438. 'filename': filename,
  439. 'destination': destination
  440. }
  441. if debug:
  442. print('Inbox queue item created')
  443. saveJson(newQueueItem, filename)
  444. return filename
  445. def _inboxPostRecipientsAdd(baseDir: str, httpPrefix: str, toList: [],
  446. recipientsDict: {},
  447. domainMatch: str, domain: str,
  448. actor: str, debug: bool) -> bool:
  449. """Given a list of post recipients (toList) from 'to' or 'cc' parameters
  450. populate a recipientsDict with the handle for each
  451. """
  452. followerRecipients = False
  453. for recipient in toList:
  454. if not recipient:
  455. continue
  456. # is this a to a local account?
  457. if domainMatch in recipient:
  458. # get the handle for the local account
  459. nickname = recipient.split(domainMatch)[1]
  460. handle = nickname+'@'+domain
  461. if os.path.isdir(baseDir + '/accounts/' + handle):
  462. recipientsDict[handle] = None
  463. else:
  464. if debug:
  465. print('DEBUG: ' + baseDir + '/accounts/' +
  466. handle + ' does not exist')
  467. else:
  468. if debug:
  469. print('DEBUG: ' + recipient + ' is not local to ' +
  470. domainMatch)
  471. print(str(toList))
  472. if recipient.endswith('followers'):
  473. if debug:
  474. print('DEBUG: followers detected as post recipients')
  475. followerRecipients = True
  476. return followerRecipients, recipientsDict
  477. def _inboxPostRecipients(baseDir: str, postJsonObject: {},
  478. httpPrefix: str, domain: str, port: int,
  479. debug: bool) -> ([], []):
  480. """Returns dictionaries containing the recipients of the given post
  481. The shared dictionary contains followers
  482. """
  483. recipientsDict = {}
  484. recipientsDictFollowers = {}
  485. if not postJsonObject.get('actor'):
  486. if debug:
  487. pprint(postJsonObject)
  488. print('WARNING: inbox post has no actor')
  489. return recipientsDict, recipientsDictFollowers
  490. if ':' in domain:
  491. domain = domain.split(':')[0]
  492. domainBase = domain
  493. domain = getFullDomain(domain, port)
  494. domainMatch = '/' + domain + '/users/'
  495. actor = postJsonObject['actor']
  496. # first get any specific people which the post is addressed to
  497. followerRecipients = False
  498. if postJsonObject.get('object'):
  499. if isinstance(postJsonObject['object'], dict):
  500. if postJsonObject['object'].get('to'):
  501. if isinstance(postJsonObject['object']['to'], list):
  502. recipientsList = postJsonObject['object']['to']
  503. else:
  504. recipientsList = [postJsonObject['object']['to']]
  505. if debug:
  506. print('DEBUG: resolving "to"')
  507. includesFollowers, recipientsDict = \
  508. _inboxPostRecipientsAdd(baseDir, httpPrefix,
  509. recipientsList,
  510. recipientsDict,
  511. domainMatch, domainBase,
  512. actor, debug)
  513. if includesFollowers:
  514. followerRecipients = True
  515. else:
  516. if debug:
  517. print('DEBUG: inbox post has no "to"')
  518. if postJsonObject['object'].get('cc'):
  519. if isinstance(postJsonObject['object']['cc'], list):
  520. recipientsList = postJsonObject['object']['cc']
  521. else:
  522. recipientsList = [postJsonObject['object']['cc']]
  523. includesFollowers, recipientsDict = \
  524. _inboxPostRecipientsAdd(baseDir, httpPrefix,
  525. recipientsList,
  526. recipientsDict,
  527. domainMatch, domainBase,
  528. actor, debug)
  529. if includesFollowers:
  530. followerRecipients = True
  531. else:
  532. if debug:
  533. print('DEBUG: inbox post has no cc')
  534. else:
  535. if debug:
  536. if isinstance(postJsonObject['object'], str):
  537. if '/statuses/' in postJsonObject['object']:
  538. print('DEBUG: inbox item is a link to a post')
  539. else:
  540. if '/users/' in postJsonObject['object']:
  541. print('DEBUG: inbox item is a link to an actor')
  542. if postJsonObject.get('to'):
  543. if isinstance(postJsonObject['to'], list):
  544. recipientsList = postJsonObject['to']
  545. else:
  546. recipientsList = [postJsonObject['to']]
  547. includesFollowers, recipientsDict = \
  548. _inboxPostRecipientsAdd(baseDir, httpPrefix,
  549. recipientsList,
  550. recipientsDict,
  551. domainMatch, domainBase,
  552. actor, debug)
  553. if includesFollowers:
  554. followerRecipients = True
  555. if postJsonObject.get('cc'):
  556. if isinstance(postJsonObject['cc'], list):
  557. recipientsList = postJsonObject['cc']
  558. else:
  559. recipientsList = [postJsonObject['cc']]
  560. includesFollowers, recipientsDict = \
  561. _inboxPostRecipientsAdd(baseDir, httpPrefix,
  562. recipientsList,
  563. recipientsDict,
  564. domainMatch, domainBase,
  565. actor, debug)
  566. if includesFollowers:
  567. followerRecipients = True
  568. if not followerRecipients:
  569. if debug:
  570. print('DEBUG: no followers were resolved')
  571. return recipientsDict, recipientsDictFollowers
  572. # now resolve the followers
  573. recipientsDictFollowers = \
  574. getFollowersOfActor(baseDir, actor, debug)
  575. return recipientsDict, recipientsDictFollowers
  576. def _receiveUndoFollow(session, baseDir: str, httpPrefix: str,
  577. port: int, messageJson: {},
  578. federationList: [],
  579. debug: bool) -> bool:
  580. if not messageJson['object'].get('actor'):
  581. if debug:
  582. print('DEBUG: follow request has no actor within object')
  583. return False
  584. if not hasUsersPath(messageJson['object']['actor']):
  585. if debug:
  586. print('DEBUG: "users" or "profile" missing ' +
  587. 'from actor within object')
  588. return False
  589. if messageJson['object']['actor'] != messageJson['actor']:
  590. if debug:
  591. print('DEBUG: actors do not match')
  592. return False
  593. nicknameFollower = \
  594. getNicknameFromActor(messageJson['object']['actor'])
  595. if not nicknameFollower:
  596. print('WARN: unable to find nickname in ' +
  597. messageJson['object']['actor'])
  598. return False
  599. domainFollower, portFollower = \
  600. getDomainFromActor(messageJson['object']['actor'])
  601. domainFollowerFull = getFullDomain(domainFollower, portFollower)
  602. nicknameFollowing = \
  603. getNicknameFromActor(messageJson['object']['object'])
  604. if not nicknameFollowing:
  605. print('WARN: unable to find nickname in ' +
  606. messageJson['object']['object'])
  607. return False
  608. domainFollowing, portFollowing = \
  609. getDomainFromActor(messageJson['object']['object'])
  610. domainFollowingFull = getFullDomain(domainFollowing, portFollowing)
  611. if unfollowerOfAccount(baseDir,
  612. nicknameFollowing, domainFollowingFull,
  613. nicknameFollower, domainFollowerFull,
  614. debug):
  615. print(nicknameFollowing + '@' + domainFollowingFull + ': '
  616. 'Follower ' + nicknameFollower + '@' + domainFollowerFull +
  617. ' was removed')
  618. return True
  619. if debug:
  620. print('DEBUG: Follower ' +
  621. nicknameFollower + '@' + domainFollowerFull +
  622. ' was not removed')
  623. return False
  624. def _receiveUndo(session, baseDir: str, httpPrefix: str,
  625. port: int, sendThreads: [], postLog: [],
  626. cachedWebfingers: {}, personCache: {},
  627. messageJson: {}, federationList: [],
  628. debug: bool) -> bool:
  629. """Receives an undo request within the POST section of HTTPServer
  630. """
  631. if not messageJson['type'].startswith('Undo'):
  632. return False
  633. if debug:
  634. print('DEBUG: Undo activity received')
  635. if not messageJson.get('actor'):
  636. if debug:
  637. print('DEBUG: follow request has no actor')
  638. return False
  639. if not hasUsersPath(messageJson['actor']):
  640. if debug:
  641. print('DEBUG: "users" or "profile" missing from actor')
  642. return False
  643. if not messageJson.get('object'):
  644. if debug:
  645. print('DEBUG: ' + messageJson['type'] + ' has no object')
  646. return False
  647. if not isinstance(messageJson['object'], dict):
  648. if debug:
  649. print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
  650. return False
  651. if not messageJson['object'].get('type'):
  652. if debug:
  653. print('DEBUG: ' + messageJson['type'] + ' has no object type')
  654. return False
  655. if not messageJson['object'].get('object'):
  656. if debug:
  657. print('DEBUG: ' + messageJson['type'] +
  658. ' has no object within object')
  659. return False
  660. if not isinstance(messageJson['object']['object'], str):
  661. if debug:
  662. print('DEBUG: ' + messageJson['type'] +
  663. ' object within object is not a string')
  664. return False
  665. if messageJson['object']['type'] == 'Follow' or \
  666. messageJson['object']['type'] == 'Join':
  667. return _receiveUndoFollow(session, baseDir, httpPrefix,
  668. port, messageJson,
  669. federationList, debug)
  670. return False
  671. def _receiveEventPost(recentPostsCache: {}, session, baseDir: str,
  672. httpPrefix: str, domain: str, port: int,
  673. sendThreads: [], postLog: [], cachedWebfingers: {},
  674. personCache: {}, messageJson: {}, federationList: [],
  675. nickname: str, debug: bool) -> bool:
  676. """Receive a mobilizon-type event activity
  677. See https://framagit.org/framasoft/mobilizon/-/blob/
  678. master/lib/federation/activity_stream/converter/event.ex
  679. """
  680. if not isEventPost(messageJson):
  681. return
  682. print('Receiving event: ' + str(messageJson['object']))
  683. handle = getFullDomain(nickname + '@' + domain, port)
  684. postId = removeIdEnding(messageJson['id']).replace('/', '#')
  685. saveEventPost(baseDir, handle, postId, messageJson['object'])
  686. def _personReceiveUpdate(baseDir: str,
  687. domain: str, port: int,
  688. updateNickname: str, updateDomain: str,
  689. updatePort: int,
  690. personJson: {}, personCache: {},
  691. debug: bool) -> bool:
  692. """Changes an actor. eg: avatar or display name change
  693. """
  694. if debug:
  695. print('Receiving actor update for ' + personJson['url'] +
  696. ' ' + str(personJson))
  697. domainFull = getFullDomain(domain, port)
  698. updateDomainFull = getFullDomain(updateDomain, updatePort)
  699. usersPaths = ('users', 'profile', 'channel', 'accounts', 'u')
  700. usersStrFound = False
  701. for usersStr in usersPaths:
  702. actor = updateDomainFull + '/' + usersStr + '/' + updateNickname
  703. if actor in personJson['id']:
  704. usersStrFound = True
  705. break
  706. if not usersStrFound:
  707. if debug:
  708. print('actor: ' + actor)
  709. print('id: ' + personJson['id'])
  710. print('DEBUG: Actor does not match id')
  711. return False
  712. if updateDomainFull == domainFull:
  713. if debug:
  714. print('DEBUG: You can only receive actor updates ' +
  715. 'for domains other than your own')
  716. return False
  717. if not personJson.get('publicKey'):
  718. if debug:
  719. print('DEBUG: actor update does not contain a public key')
  720. return False
  721. if not personJson['publicKey'].get('publicKeyPem'):
  722. if debug:
  723. print('DEBUG: actor update does not contain a public key Pem')
  724. return False
  725. actorFilename = baseDir + '/cache/actors/' + \
  726. personJson['id'].replace('/', '#') + '.json'
  727. # check that the public keys match.
  728. # If they don't then this may be a nefarious attempt to hack an account
  729. idx = personJson['id']
  730. if personCache.get(idx):
  731. if personCache[idx]['actor']['publicKey']['publicKeyPem'] != \
  732. personJson['publicKey']['publicKeyPem']:
  733. if debug:
  734. print('WARN: Public key does not match when updating actor')
  735. return False
  736. else:
  737. if os.path.isfile(actorFilename):
  738. existingPersonJson = loadJson(actorFilename)
  739. if existingPersonJson:
  740. if existingPersonJson['publicKey']['publicKeyPem'] != \
  741. personJson['publicKey']['publicKeyPem']:
  742. if debug:
  743. print('WARN: Public key does not match ' +
  744. 'cached actor when updating')
  745. return False
  746. # save to cache in memory
  747. storePersonInCache(baseDir, personJson['id'], personJson,
  748. personCache, True)
  749. # save to cache on file
  750. if saveJson(personJson, actorFilename):
  751. if debug:
  752. print('actor updated for ' + personJson['id'])
  753. # remove avatar if it exists so that it will be refreshed later
  754. # when a timeline is constructed
  755. actorStr = personJson['id'].replace('/', '-')
  756. removeAvatarFromCache(baseDir, actorStr)
  757. return True
  758. def _receiveUpdateToQuestion(recentPostsCache: {}, messageJson: {},
  759. baseDir: str,
  760. nickname: str, domain: str) -> None:
  761. """Updating a question as new votes arrive
  762. """
  763. # message url of the question
  764. if not messageJson.get('id'):
  765. return
  766. if not messageJson.get('actor'):
  767. return
  768. messageId = removeIdEnding(messageJson['id'])
  769. if '#' in messageId:
  770. messageId = messageId.split('#', 1)[0]
  771. # find the question post
  772. postFilename = locatePost(baseDir, nickname, domain, messageId)
  773. if not postFilename:
  774. return
  775. # load the json for the question
  776. postJsonObject = loadJson(postFilename, 1)
  777. if not postJsonObject:
  778. return
  779. if not postJsonObject.get('actor'):
  780. return
  781. # does the actor match?
  782. if postJsonObject['actor'] != messageJson['actor']:
  783. return
  784. saveJson(messageJson, postFilename)
  785. # ensure that the cached post is removed if it exists, so
  786. # that it then will be recreated
  787. cachedPostFilename = \
  788. getCachedPostFilename(baseDir, nickname, domain, messageJson)
  789. if cachedPostFilename:
  790. if os.path.isfile(cachedPostFilename):
  791. os.remove(cachedPostFilename)
  792. # remove from memory cache
  793. removePostFromCache(messageJson, recentPostsCache)
  794. def _receiveUpdate(recentPostsCache: {}, session, baseDir: str,
  795. httpPrefix: str, domain: str, port: int,
  796. sendThreads: [], postLog: [], cachedWebfingers: {},
  797. personCache: {}, messageJson: {}, federationList: [],
  798. nickname: str, debug: bool) -> bool:
  799. """Receives an Update activity within the POST section of HTTPServer
  800. """
  801. if messageJson['type'] != 'Update':
  802. return False
  803. if not messageJson.get('actor'):
  804. if debug:
  805. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  806. return False
  807. if not messageJson.get('object'):
  808. if debug:
  809. print('DEBUG: ' + messageJson['type'] + ' has no object')
  810. return False
  811. if not isinstance(messageJson['object'], dict):
  812. if debug:
  813. print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
  814. return False
  815. if not messageJson['object'].get('type'):
  816. if debug:
  817. print('DEBUG: ' + messageJson['type'] + ' object has no type')
  818. return False
  819. if not hasUsersPath(messageJson['actor']):
  820. if debug:
  821. print('DEBUG: "users" or "profile" missing from actor in ' +
  822. messageJson['type'])
  823. return False
  824. if messageJson['object']['type'] == 'Question':
  825. _receiveUpdateToQuestion(recentPostsCache, messageJson,
  826. baseDir, nickname, domain)
  827. if debug:
  828. print('DEBUG: Question update was received')
  829. return True
  830. if messageJson['type'] == 'Person':
  831. if messageJson.get('url') and messageJson.get('id'):
  832. if debug:
  833. print('Request to update actor unwrapped: ' +
  834. str(messageJson))
  835. updateNickname = getNicknameFromActor(messageJson['id'])
  836. if updateNickname:
  837. updateDomain, updatePort = \
  838. getDomainFromActor(messageJson['id'])
  839. if _personReceiveUpdate(baseDir, domain, port,
  840. updateNickname, updateDomain,
  841. updatePort, messageJson,
  842. personCache, debug):
  843. if debug:
  844. print('DEBUG: ' +
  845. 'Unwrapped profile update was received for ' +
  846. messageJson['url'])
  847. return True
  848. if messageJson['object']['type'] == 'Person' or \
  849. messageJson['object']['type'] == 'Application' or \
  850. messageJson['object']['type'] == 'Group' or \
  851. messageJson['object']['type'] == 'Service':
  852. if messageJson['object'].get('url') and \
  853. messageJson['object'].get('id'):
  854. if debug:
  855. print('Request to update actor: ' + str(messageJson))
  856. updateNickname = getNicknameFromActor(messageJson['actor'])
  857. if updateNickname:
  858. updateDomain, updatePort = \
  859. getDomainFromActor(messageJson['actor'])
  860. if _personReceiveUpdate(baseDir,
  861. domain, port,
  862. updateNickname, updateDomain,
  863. updatePort,
  864. messageJson['object'],
  865. personCache, debug):
  866. if debug:
  867. print('DEBUG: Profile update was received for ' +
  868. messageJson['object']['url'])
  869. return True
  870. return False
  871. def _receiveLike(recentPostsCache: {},
  872. session, handle: str, isGroup: bool, baseDir: str,
  873. httpPrefix: str, domain: str, port: int,
  874. onionDomain: str,
  875. sendThreads: [], postLog: [], cachedWebfingers: {},
  876. personCache: {}, messageJson: {}, federationList: [],
  877. debug: bool) -> bool:
  878. """Receives a Like activity within the POST section of HTTPServer
  879. """
  880. if messageJson['type'] != 'Like':
  881. return False
  882. if not messageJson.get('actor'):
  883. if debug:
  884. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  885. return False
  886. if not messageJson.get('object'):
  887. if debug:
  888. print('DEBUG: ' + messageJson['type'] + ' has no object')
  889. return False
  890. if not isinstance(messageJson['object'], str):
  891. if debug:
  892. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  893. return False
  894. if not messageJson.get('to'):
  895. if debug:
  896. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  897. return False
  898. if not hasUsersPath(messageJson['actor']):
  899. if debug:
  900. print('DEBUG: "users" or "profile" missing from actor in ' +
  901. messageJson['type'])
  902. return False
  903. if '/statuses/' not in messageJson['object']:
  904. if debug:
  905. print('DEBUG: "statuses" missing from object in ' +
  906. messageJson['type'])
  907. return False
  908. if not os.path.isdir(baseDir + '/accounts/' + handle):
  909. print('DEBUG: unknown recipient of like - ' + handle)
  910. # if this post in the outbox of the person?
  911. handleName = handle.split('@')[0]
  912. handleDom = handle.split('@')[1]
  913. postFilename = locatePost(baseDir, handleName, handleDom,
  914. messageJson['object'])
  915. if not postFilename:
  916. if debug:
  917. print('DEBUG: post not found in inbox or outbox')
  918. print(messageJson['object'])
  919. return True
  920. if debug:
  921. print('DEBUG: liked post found in inbox')
  922. handleName = handle.split('@')[0]
  923. handleDom = handle.split('@')[1]
  924. if not _alreadyLiked(baseDir,
  925. handleName, handleDom,
  926. messageJson['object'],
  927. messageJson['actor']):
  928. _likeNotify(baseDir, domain, onionDomain, handle,
  929. messageJson['actor'], messageJson['object'])
  930. updateLikesCollection(recentPostsCache, baseDir, postFilename,
  931. messageJson['object'],
  932. messageJson['actor'], domain, debug)
  933. return True
  934. def _receiveUndoLike(recentPostsCache: {},
  935. session, handle: str, isGroup: bool, baseDir: str,
  936. httpPrefix: str, domain: str, port: int,
  937. sendThreads: [], postLog: [], cachedWebfingers: {},
  938. personCache: {}, messageJson: {}, federationList: [],
  939. debug: bool) -> bool:
  940. """Receives an undo like activity within the POST section of HTTPServer
  941. """
  942. if messageJson['type'] != 'Undo':
  943. return False
  944. if not messageJson.get('actor'):
  945. return False
  946. if not messageJson.get('object'):
  947. return False
  948. if not isinstance(messageJson['object'], dict):
  949. return False
  950. if not messageJson['object'].get('type'):
  951. return False
  952. if messageJson['object']['type'] != 'Like':
  953. return False
  954. if not messageJson['object'].get('object'):
  955. if debug:
  956. print('DEBUG: ' + messageJson['type'] + ' like has no object')
  957. return False
  958. if not isinstance(messageJson['object']['object'], str):
  959. if debug:
  960. print('DEBUG: ' + messageJson['type'] +
  961. ' like object is not a string')
  962. return False
  963. if not hasUsersPath(messageJson['actor']):
  964. if debug:
  965. print('DEBUG: "users" or "profile" missing from actor in ' +
  966. messageJson['type'] + ' like')
  967. return False
  968. if '/statuses/' not in messageJson['object']['object']:
  969. if debug:
  970. print('DEBUG: "statuses" missing from like object in ' +
  971. messageJson['type'])
  972. return False
  973. if not os.path.isdir(baseDir + '/accounts/' + handle):
  974. print('DEBUG: unknown recipient of undo like - ' + handle)
  975. # if this post in the outbox of the person?
  976. handleName = handle.split('@')[0]
  977. handleDom = handle.split('@')[1]
  978. postFilename = \
  979. locatePost(baseDir, handleName, handleDom,
  980. messageJson['object']['object'])
  981. if not postFilename:
  982. if debug:
  983. print('DEBUG: unliked post not found in inbox or outbox')
  984. print(messageJson['object']['object'])
  985. return True
  986. if debug:
  987. print('DEBUG: liked post found in inbox. Now undoing.')
  988. undoLikesCollectionEntry(recentPostsCache, baseDir, postFilename,
  989. messageJson['object'],
  990. messageJson['actor'], domain, debug)
  991. return True
  992. def _receiveBookmark(recentPostsCache: {},
  993. session, handle: str, isGroup: bool, baseDir: str,
  994. httpPrefix: str, domain: str, port: int,
  995. sendThreads: [], postLog: [], cachedWebfingers: {},
  996. personCache: {}, messageJson: {}, federationList: [],
  997. debug: bool) -> bool:
  998. """Receives a bookmark activity within the POST section of HTTPServer
  999. """
  1000. if not messageJson.get('type'):
  1001. return False
  1002. if messageJson['type'] != 'Add':
  1003. return False
  1004. if not messageJson.get('actor'):
  1005. if debug:
  1006. print('DEBUG: no actor in inbox bookmark Add')
  1007. return False
  1008. if not messageJson.get('object'):
  1009. if debug:
  1010. print('DEBUG: no object in inbox bookmark Add')
  1011. return False
  1012. if not messageJson.get('target'):
  1013. if debug:
  1014. print('DEBUG: no target in inbox bookmark Add')
  1015. return False
  1016. if not isinstance(messageJson['object'], dict):
  1017. if debug:
  1018. print('DEBUG: inbox bookmark Add object is not string')
  1019. return False
  1020. if not messageJson['object'].get('type'):
  1021. if debug:
  1022. print('DEBUG: no object type in inbox bookmark Add')
  1023. return False
  1024. if not isinstance(messageJson['target'], str):
  1025. if debug:
  1026. print('DEBUG: inbox bookmark Add target is not string')
  1027. return False
  1028. domainFull = getFullDomain(domain, port)
  1029. nickname = handle.split('@')[0]
  1030. if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
  1031. if debug:
  1032. print('DEBUG: inbox bookmark Add unexpected actor')
  1033. return False
  1034. if not messageJson['target'].endswith(messageJson['actor'] +
  1035. '/tlbookmarks'):
  1036. if debug:
  1037. print('DEBUG: inbox bookmark Add target invalid ' +
  1038. messageJson['target'])
  1039. return False
  1040. if messageJson['object']['type'] != 'Document':
  1041. if debug:
  1042. print('DEBUG: inbox bookmark Add type is not Document')
  1043. return False
  1044. if not messageJson['object'].get('url'):
  1045. if debug:
  1046. print('DEBUG: inbox bookmark Add missing url')
  1047. return False
  1048. if '/statuses/' not in messageJson['object']['url']:
  1049. if debug:
  1050. print('DEBUG: inbox bookmark Add missing statuses un url')
  1051. return False
  1052. if debug:
  1053. print('DEBUG: c2s inbox bookmark Add request arrived in outbox')
  1054. messageUrl = removeIdEnding(messageJson['object']['url'])
  1055. if ':' in domain:
  1056. domain = domain.split(':')[0]
  1057. postFilename = locatePost(baseDir, nickname, domain, messageUrl)
  1058. if not postFilename:
  1059. if debug:
  1060. print('DEBUG: c2s inbox like post not found in inbox or outbox')
  1061. print(messageUrl)
  1062. return True
  1063. updateBookmarksCollection(recentPostsCache, baseDir, postFilename,
  1064. messageJson['object']['url'],
  1065. messageJson['actor'], domain, debug)
  1066. return True
  1067. def _receiveUndoBookmark(recentPostsCache: {},
  1068. session, handle: str, isGroup: bool, baseDir: str,
  1069. httpPrefix: str, domain: str, port: int,
  1070. sendThreads: [], postLog: [], cachedWebfingers: {},
  1071. personCache: {}, messageJson: {}, federationList: [],
  1072. debug: bool) -> bool:
  1073. """Receives an undo bookmark activity within the POST section of HTTPServer
  1074. """
  1075. if not messageJson.get('type'):
  1076. return False
  1077. if messageJson['type'] != 'Remove':
  1078. return False
  1079. if not messageJson.get('actor'):
  1080. if debug:
  1081. print('DEBUG: no actor in inbox undo bookmark Remove')
  1082. return False
  1083. if not messageJson.get('object'):
  1084. if debug:
  1085. print('DEBUG: no object in inbox undo bookmark Remove')
  1086. return False
  1087. if not messageJson.get('target'):
  1088. if debug:
  1089. print('DEBUG: no target in inbox undo bookmark Remove')
  1090. return False
  1091. if not isinstance(messageJson['object'], dict):
  1092. if debug:
  1093. print('DEBUG: inbox Remove bookmark object is not dict')
  1094. return False
  1095. if not messageJson['object'].get('type'):
  1096. if debug:
  1097. print('DEBUG: no object type in inbox bookmark Remove')
  1098. return False
  1099. if not isinstance(messageJson['target'], str):
  1100. if debug:
  1101. print('DEBUG: inbox Remove bookmark target is not string')
  1102. return False
  1103. domainFull = getFullDomain(domain, port)
  1104. nickname = handle.split('@')[0]
  1105. if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
  1106. if debug:
  1107. print('DEBUG: inbox undo bookmark Remove unexpected actor')
  1108. return False
  1109. if not messageJson['target'].endswith(messageJson['actor'] +
  1110. '/tlbookmarks'):
  1111. if debug:
  1112. print('DEBUG: inbox undo bookmark Remove target invalid ' +
  1113. messageJson['target'])
  1114. return False
  1115. if messageJson['object']['type'] != 'Document':
  1116. if debug:
  1117. print('DEBUG: inbox undo bookmark Remove type is not Document')
  1118. return False
  1119. if not messageJson['object'].get('url'):
  1120. if debug:
  1121. print('DEBUG: inbox undo bookmark Remove missing url')
  1122. return False
  1123. if '/statuses/' not in messageJson['object']['url']:
  1124. if debug:
  1125. print('DEBUG: inbox undo bookmark Remove missing statuses un url')
  1126. return False
  1127. if debug:
  1128. print('DEBUG: c2s inbox Remove bookmark ' +
  1129. 'request arrived in outbox')
  1130. messageUrl = removeIdEnding(messageJson['object']['url'])
  1131. if ':' in domain:
  1132. domain = domain.split(':')[0]
  1133. postFilename = locatePost(baseDir, nickname, domain, messageUrl)
  1134. if not postFilename:
  1135. if debug:
  1136. print('DEBUG: c2s inbox like post not found in inbox or outbox')
  1137. print(messageUrl)
  1138. return True
  1139. undoBookmarksCollectionEntry(recentPostsCache, baseDir, postFilename,
  1140. messageJson['object']['url'],
  1141. messageJson['actor'], domain, debug)
  1142. return True
  1143. def _receiveDelete(session, handle: str, isGroup: bool, baseDir: str,
  1144. httpPrefix: str, domain: str, port: int,
  1145. sendThreads: [], postLog: [], cachedWebfingers: {},
  1146. personCache: {}, messageJson: {}, federationList: [],
  1147. debug: bool, allowDeletion: bool,
  1148. recentPostsCache: {}) -> bool:
  1149. """Receives a Delete activity within the POST section of HTTPServer
  1150. """
  1151. if messageJson['type'] != 'Delete':
  1152. return False
  1153. if not messageJson.get('actor'):
  1154. if debug:
  1155. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  1156. return False
  1157. if debug:
  1158. print('DEBUG: Delete activity arrived')
  1159. if not messageJson.get('object'):
  1160. if debug:
  1161. print('DEBUG: ' + messageJson['type'] + ' has no object')
  1162. return False
  1163. if not isinstance(messageJson['object'], str):
  1164. if debug:
  1165. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  1166. return False
  1167. domainFull = getFullDomain(domain, port)
  1168. deletePrefix = httpPrefix + '://' + domainFull + '/'
  1169. if (not allowDeletion and
  1170. (not messageJson['object'].startswith(deletePrefix) or
  1171. not messageJson['actor'].startswith(deletePrefix))):
  1172. if debug:
  1173. print('DEBUG: delete not permitted from other instances')
  1174. return False
  1175. if not messageJson.get('to'):
  1176. if debug:
  1177. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  1178. return False
  1179. if not hasUsersPath(messageJson['actor']):
  1180. if debug:
  1181. print('DEBUG: ' +
  1182. '"users" or "profile" missing from actor in ' +
  1183. messageJson['type'])
  1184. return False
  1185. if '/statuses/' not in messageJson['object']:
  1186. if debug:
  1187. print('DEBUG: "statuses" missing from object in ' +
  1188. messageJson['type'])
  1189. return False
  1190. if messageJson['actor'] not in messageJson['object']:
  1191. if debug:
  1192. print('DEBUG: actor is not the owner of the post to be deleted')
  1193. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1194. print('DEBUG: unknown recipient of like - ' + handle)
  1195. # if this post in the outbox of the person?
  1196. messageId = removeIdEnding(messageJson['object'])
  1197. removeModerationPostFromIndex(baseDir, messageId, debug)
  1198. handleNickname = handle.split('@')[0]
  1199. handleDomain = handle.split('@')[1]
  1200. postFilename = locatePost(baseDir, handleNickname,
  1201. handleDomain, messageId)
  1202. if not postFilename:
  1203. if debug:
  1204. print('DEBUG: delete post not found in inbox or outbox')
  1205. print(messageId)
  1206. return True
  1207. deletePost(baseDir, httpPrefix, handleNickname,
  1208. handleDomain, postFilename, debug,
  1209. recentPostsCache)
  1210. if debug:
  1211. print('DEBUG: post deleted - ' + postFilename)
  1212. # also delete any local blogs saved to the news actor
  1213. if handleNickname != 'news' and handleDomain == domainFull:
  1214. postFilename = locatePost(baseDir, 'news',
  1215. handleDomain, messageId)
  1216. if postFilename:
  1217. deletePost(baseDir, httpPrefix, 'news',
  1218. handleDomain, postFilename, debug,
  1219. recentPostsCache)
  1220. if debug:
  1221. print('DEBUG: blog post deleted - ' + postFilename)
  1222. return True
  1223. def _receiveAnnounce(recentPostsCache: {},
  1224. session, handle: str, isGroup: bool, baseDir: str,
  1225. httpPrefix: str,
  1226. domain: str, onionDomain: str, port: int,
  1227. sendThreads: [], postLog: [], cachedWebfingers: {},
  1228. personCache: {}, messageJson: {}, federationList: [],
  1229. debug: bool, translate: {},
  1230. YTReplacementDomain: str,
  1231. allowLocalNetworkAccess: bool,
  1232. themeName: str) -> bool:
  1233. """Receives an announce activity within the POST section of HTTPServer
  1234. """
  1235. if messageJson['type'] != 'Announce':
  1236. return False
  1237. if '@' not in handle:
  1238. if debug:
  1239. print('DEBUG: bad handle ' + handle)
  1240. return False
  1241. if not messageJson.get('actor'):
  1242. if debug:
  1243. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  1244. return False
  1245. if debug:
  1246. print('DEBUG: receiving announce on ' + handle)
  1247. if not messageJson.get('object'):
  1248. if debug:
  1249. print('DEBUG: ' + messageJson['type'] + ' has no object')
  1250. return False
  1251. if not isinstance(messageJson['object'], str):
  1252. if debug:
  1253. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  1254. return False
  1255. if not messageJson.get('to'):
  1256. if debug:
  1257. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  1258. return False
  1259. if not hasUsersPath(messageJson['actor']):
  1260. if debug:
  1261. print('DEBUG: ' +
  1262. '"users" or "profile" missing from actor in ' +
  1263. messageJson['type'])
  1264. return False
  1265. if not hasUsersPath(messageJson['object']):
  1266. if debug:
  1267. print('DEBUG: ' +
  1268. '"users", "channel" or "profile" missing in ' +
  1269. messageJson['type'])
  1270. return False
  1271. prefixes = getProtocolPrefixes()
  1272. # is the domain of the announce actor blocked?
  1273. objectDomain = messageJson['object']
  1274. for prefix in prefixes:
  1275. objectDomain = objectDomain.replace(prefix, '')
  1276. if '/' in objectDomain:
  1277. objectDomain = objectDomain.split('/')[0]
  1278. if isBlockedDomain(baseDir, objectDomain):
  1279. if debug:
  1280. print('DEBUG: announced domain is blocked')
  1281. return False
  1282. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1283. print('DEBUG: unknown recipient of announce - ' + handle)
  1284. # is the announce actor blocked?
  1285. nickname = handle.split('@')[0]
  1286. actorNickname = getNicknameFromActor(messageJson['actor'])
  1287. actorDomain, actorPort = getDomainFromActor(messageJson['actor'])
  1288. if isBlocked(baseDir, nickname, domain, actorNickname, actorDomain):
  1289. print('Receive announce blocked for actor: ' +
  1290. actorNickname + '@' + actorDomain)
  1291. return False
  1292. # is this post in the outbox of the person?
  1293. postFilename = locatePost(baseDir, nickname, domain,
  1294. messageJson['object'])
  1295. if not postFilename:
  1296. if debug:
  1297. print('DEBUG: announce post not found in inbox or outbox')
  1298. print(messageJson['object'])
  1299. return True
  1300. updateAnnounceCollection(recentPostsCache, baseDir, postFilename,
  1301. messageJson['actor'], domain, debug)
  1302. if debug:
  1303. print('DEBUG: Downloading announce post ' + messageJson['actor'] +
  1304. ' -> ' + messageJson['object'])
  1305. postJsonObject = downloadAnnounce(session, baseDir,
  1306. httpPrefix,
  1307. nickname, domain,
  1308. messageJson,
  1309. __version__, translate,
  1310. YTReplacementDomain,
  1311. allowLocalNetworkAccess,
  1312. recentPostsCache, debug)
  1313. if not postJsonObject:
  1314. notInOnion = True
  1315. if onionDomain:
  1316. if onionDomain in messageJson['object']:
  1317. notInOnion = False
  1318. if domain not in messageJson['object'] and notInOnion:
  1319. if os.path.isfile(postFilename):
  1320. # if the announce can't be downloaded then remove it
  1321. os.remove(postFilename)
  1322. else:
  1323. if debug:
  1324. print('DEBUG: Announce post downloaded for ' +
  1325. messageJson['actor'] + ' -> ' + messageJson['object'])
  1326. storeHashTags(baseDir, nickname, postJsonObject)
  1327. # Try to obtain the actor for this person
  1328. # so that their avatar can be shown
  1329. lookupActor = None
  1330. if postJsonObject.get('attributedTo'):
  1331. if isinstance(postJsonObject['attributedTo'], str):
  1332. lookupActor = postJsonObject['attributedTo']
  1333. else:
  1334. if postJsonObject.get('object'):
  1335. if isinstance(postJsonObject['object'], dict):
  1336. if postJsonObject['object'].get('attributedTo'):
  1337. attrib = postJsonObject['object']['attributedTo']
  1338. if isinstance(attrib, str):
  1339. lookupActor = attrib
  1340. if lookupActor:
  1341. if hasUsersPath(lookupActor):
  1342. if '/statuses/' in lookupActor:
  1343. lookupActor = lookupActor.split('/statuses/')[0]
  1344. if isRecentPost(postJsonObject):
  1345. if not os.path.isfile(postFilename + '.tts'):
  1346. domainFull = getFullDomain(domain, port)
  1347. updateSpeaker(baseDir, httpPrefix,
  1348. nickname, domain, domainFull,
  1349. postJsonObject, personCache,
  1350. translate, lookupActor,
  1351. themeName)
  1352. ttsFile = open(postFilename + '.tts', "w+")
  1353. if ttsFile:
  1354. ttsFile.write('\n')
  1355. ttsFile.close()
  1356. if debug:
  1357. print('DEBUG: Obtaining actor for announce post ' +
  1358. lookupActor)
  1359. for tries in range(6):
  1360. pubKey = \
  1361. getPersonPubKey(baseDir, session, lookupActor,
  1362. personCache, debug,
  1363. __version__, httpPrefix,
  1364. domain, onionDomain)
  1365. if pubKey:
  1366. if debug:
  1367. print('DEBUG: public key obtained for announce: ' +
  1368. lookupActor)
  1369. break
  1370. if debug:
  1371. print('DEBUG: Retry ' + str(tries + 1) +
  1372. ' obtaining actor for ' + lookupActor)
  1373. time.sleep(5)
  1374. if debug:
  1375. print('DEBUG: announced/repeated post arrived in inbox')
  1376. return True
  1377. def _receiveUndoAnnounce(recentPostsCache: {},
  1378. session, handle: str, isGroup: bool, baseDir: str,
  1379. httpPrefix: str, domain: str, port: int,
  1380. sendThreads: [], postLog: [], cachedWebfingers: {},
  1381. personCache: {}, messageJson: {}, federationList: [],
  1382. debug: bool) -> bool:
  1383. """Receives an undo announce activity within the POST section of HTTPServer
  1384. """
  1385. if messageJson['type'] != 'Undo':
  1386. return False
  1387. if not messageJson.get('actor'):
  1388. return False
  1389. if not messageJson.get('object'):
  1390. return False
  1391. if not isinstance(messageJson['object'], dict):
  1392. return False
  1393. if not messageJson['object'].get('object'):
  1394. return False
  1395. if not isinstance(messageJson['object']['object'], str):
  1396. return False
  1397. if messageJson['object']['type'] != 'Announce':
  1398. return False
  1399. if not hasUsersPath(messageJson['actor']):
  1400. if debug:
  1401. print('DEBUG: "users" or "profile" missing from actor in ' +
  1402. messageJson['type'] + ' announce')
  1403. return False
  1404. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1405. print('DEBUG: unknown recipient of undo announce - ' + handle)
  1406. # if this post in the outbox of the person?
  1407. handleName = handle.split('@')[0]
  1408. handleDom = handle.split('@')[1]
  1409. postFilename = locatePost(baseDir, handleName, handleDom,
  1410. messageJson['object']['object'])
  1411. if not postFilename:
  1412. if debug:
  1413. print('DEBUG: undo announce post not found in inbox or outbox')
  1414. print(messageJson['object']['object'])
  1415. return True
  1416. if debug:
  1417. print('DEBUG: announced/repeated post to be undone found in inbox')
  1418. postJsonObject = loadJson(postFilename)
  1419. if postJsonObject:
  1420. if not postJsonObject.get('type'):
  1421. if postJsonObject['type'] != 'Announce':
  1422. if debug:
  1423. print("DEBUG: Attempt to undo something " +
  1424. "which isn't an announcement")
  1425. return False
  1426. undoAnnounceCollectionEntry(recentPostsCache, baseDir, postFilename,
  1427. messageJson['actor'], domain, debug)
  1428. if os.path.isfile(postFilename):
  1429. os.remove(postFilename)
  1430. return True
  1431. def jsonPostAllowsComments(postJsonObject: {}) -> bool:
  1432. """Returns true if the given post allows comments/replies
  1433. """
  1434. if 'commentsEnabled' in postJsonObject:
  1435. return postJsonObject['commentsEnabled']
  1436. if postJsonObject.get('object'):
  1437. if not isinstance(postJsonObject['object'], dict):
  1438. return False
  1439. if 'commentsEnabled' in postJsonObject['object']:
  1440. return postJsonObject['object']['commentsEnabled']
  1441. return True
  1442. def _postAllowsComments(postFilename: str) -> bool:
  1443. """Returns true if the given post allows comments/replies
  1444. """
  1445. postJsonObject = loadJson(postFilename)
  1446. if not postJsonObject:
  1447. return False
  1448. return jsonPostAllowsComments(postJsonObject)
  1449. def populateReplies(baseDir: str, httpPrefix: str, domain: str,
  1450. messageJson: {}, maxReplies: int, debug: bool) -> bool:
  1451. """Updates the list of replies for a post on this domain if
  1452. a reply to it arrives
  1453. """
  1454. if not messageJson.get('id'):
  1455. return False
  1456. if not messageJson.get('object'):
  1457. return False
  1458. if not isinstance(messageJson['object'], dict):
  1459. return False
  1460. if not messageJson['object'].get('inReplyTo'):
  1461. return False
  1462. if not messageJson['object'].get('to'):
  1463. return False
  1464. replyTo = messageJson['object']['inReplyTo']
  1465. if not isinstance(replyTo, str):
  1466. return False
  1467. if debug:
  1468. print('DEBUG: post contains a reply')
  1469. # is this a reply to a post on this domain?
  1470. if not replyTo.startswith(httpPrefix + '://' + domain + '/'):
  1471. if debug:
  1472. print('DEBUG: post is a reply to another not on this domain')
  1473. print(replyTo)
  1474. print('Expected: ' + httpPrefix + '://' + domain + '/')
  1475. return False
  1476. replyToNickname = getNicknameFromActor(replyTo)
  1477. if not replyToNickname:
  1478. print('DEBUG: no nickname found for ' + replyTo)
  1479. return False
  1480. replyToDomain, replyToPort = getDomainFromActor(replyTo)
  1481. if not replyToDomain:
  1482. if debug:
  1483. print('DEBUG: no domain found for ' + replyTo)
  1484. return False
  1485. postFilename = locatePost(baseDir, replyToNickname,
  1486. replyToDomain, replyTo)
  1487. if not postFilename:
  1488. if debug:
  1489. print('DEBUG: post may have expired - ' + replyTo)
  1490. return False
  1491. # TODO store replies collection
  1492. # replyItem = {
  1493. # "type": "Document",
  1494. # "url": replyTo
  1495. # }
  1496. # if not messageJson['object'].get('replies'):
  1497. # messageJson['object']['replies'] = {
  1498. # "items": [replyItem]
  1499. # }
  1500. # else:
  1501. # found = False
  1502. # for item in messageJson['object']['replies']['items']:
  1503. # if item['url'] == replyTo:
  1504. # found = True
  1505. # break
  1506. # if not found:
  1507. # messageJson['object']['replies']['items'].append(replyItem)
  1508. #
  1509. if not _postAllowsComments(postFilename):
  1510. if debug:
  1511. print('DEBUG: post does not allow comments - ' + replyTo)
  1512. return False
  1513. # populate a text file containing the ids of replies
  1514. postRepliesFilename = postFilename.replace('.json', '.replies')
  1515. messageId = removeIdEnding(messageJson['id'])
  1516. if os.path.isfile(postRepliesFilename):
  1517. numLines = sum(1 for line in open(postRepliesFilename))
  1518. if numLines > maxReplies:
  1519. return False
  1520. if messageId not in open(postRepliesFilename).read():
  1521. repliesFile = open(postRepliesFilename, 'a+')
  1522. if repliesFile:
  1523. repliesFile.write(messageId + '\n')
  1524. repliesFile.close()
  1525. else:
  1526. repliesFile = open(postRepliesFilename, 'w+')
  1527. if repliesFile:
  1528. repliesFile.write(messageId + '\n')
  1529. repliesFile.close()
  1530. return True
  1531. def _estimateNumberOfMentions(content: str) -> int:
  1532. """Returns a rough estimate of the number of mentions
  1533. """
  1534. return int(content.count('@') / 2)
  1535. def _estimateNumberOfEmoji(content: str) -> int:
  1536. """Returns a rough estimate of the number of emoji
  1537. """
  1538. return int(content.count(':') / 2)
  1539. def _validPostContent(baseDir: str, nickname: str, domain: str,
  1540. messageJson: {}, maxMentions: int, maxEmoji: int,
  1541. allowLocalNetworkAccess: bool, debug: bool) -> bool:
  1542. """Is the content of a received post valid?
  1543. Check for bad html
  1544. Check for hellthreads
  1545. Check number of tags is reasonable
  1546. """
  1547. if not messageJson.get('object'):
  1548. return True
  1549. if not isinstance(messageJson['object'], dict):
  1550. return True
  1551. if not messageJson['object'].get('content'):
  1552. return True
  1553. if not messageJson['object'].get('published'):
  1554. return False
  1555. if 'T' not in messageJson['object']['published']:
  1556. return False
  1557. if 'Z' not in messageJson['object']['published']:
  1558. return False
  1559. if not validPostDate(messageJson['object']['published'], 90, debug):
  1560. return False
  1561. if messageJson['object'].get('summary'):
  1562. summary = messageJson['object']['summary']
  1563. if not isinstance(summary, str):
  1564. print('WARN: content warning is not a string')
  1565. return False
  1566. if summary != validContentWarning(summary):
  1567. print('WARN: invalid content warning ' + summary)
  1568. return False
  1569. if isGitPatch(baseDir, nickname, domain,
  1570. messageJson['object']['type'],
  1571. messageJson['object']['summary'],
  1572. messageJson['object']['content']):
  1573. return True
  1574. if dangerousMarkup(messageJson['object']['content'],
  1575. allowLocalNetworkAccess):
  1576. if messageJson['object'].get('id'):
  1577. print('REJECT ARBITRARY HTML: ' + messageJson['object']['id'])
  1578. print('REJECT ARBITRARY HTML: bad string in post - ' +
  1579. messageJson['object']['content'])
  1580. return False
  1581. # check (rough) number of mentions
  1582. mentionsEst = _estimateNumberOfMentions(messageJson['object']['content'])
  1583. if mentionsEst > maxMentions:
  1584. if messageJson['object'].get('id'):
  1585. print('REJECT HELLTHREAD: ' + messageJson['object']['id'])
  1586. print('REJECT HELLTHREAD: Too many mentions in post - ' +
  1587. messageJson['object']['content'])
  1588. return False
  1589. if _estimateNumberOfEmoji(messageJson['object']['content']) > maxEmoji:
  1590. if messageJson['object'].get('id'):
  1591. print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id'])
  1592. print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' +
  1593. messageJson['object']['content'])
  1594. return False
  1595. # check number of tags
  1596. if messageJson['object'].get('tag'):
  1597. if not isinstance(messageJson['object']['tag'], list):
  1598. messageJson['object']['tag'] = []
  1599. else:
  1600. if len(messageJson['object']['tag']) > int(maxMentions * 2):
  1601. if messageJson['object'].get('id'):
  1602. print('REJECT: ' + messageJson['object']['id'])
  1603. print('REJECT: Too many tags in post - ' +
  1604. messageJson['object']['tag'])
  1605. return False
  1606. # check for filtered content
  1607. if isFiltered(baseDir, nickname, domain,
  1608. messageJson['object']['content']):
  1609. print('REJECT: content filtered')
  1610. return False
  1611. if messageJson['object'].get('inReplyTo'):
  1612. if isinstance(messageJson['object']['inReplyTo'], str):
  1613. originalPostId = messageJson['object']['inReplyTo']
  1614. postPostFilename = locatePost(baseDir, nickname, domain,
  1615. originalPostId)
  1616. if postPostFilename:
  1617. if not _postAllowsComments(postPostFilename):
  1618. print('REJECT: reply to post which does not ' +
  1619. 'allow comments: ' + originalPostId)
  1620. return False
  1621. if debug:
  1622. print('ACCEPT: post content is valid')
  1623. return True
  1624. def _obtainAvatarForReplyPost(session, baseDir: str, httpPrefix: str,
  1625. domain: str, onionDomain: str, personCache: {},
  1626. postJsonObject: {}, debug: bool) -> None:
  1627. """Tries to obtain the actor for the person being replied to
  1628. so that their avatar can later be shown
  1629. """
  1630. if not postJsonObject.get('object'):
  1631. return
  1632. if not isinstance(postJsonObject['object'], dict):
  1633. return
  1634. if not postJsonObject['object'].get('inReplyTo'):
  1635. return
  1636. lookupActor = postJsonObject['object']['inReplyTo']
  1637. if not lookupActor:
  1638. return
  1639. if not isinstance(lookupActor, str):
  1640. return
  1641. if not hasUsersPath(lookupActor):
  1642. return
  1643. if '/statuses/' in lookupActor:
  1644. lookupActor = lookupActor.split('/statuses/')[0]
  1645. if debug:
  1646. print('DEBUG: Obtaining actor for reply post ' + lookupActor)
  1647. for tries in range(6):
  1648. pubKey = \
  1649. getPersonPubKey(baseDir, session, lookupActor,
  1650. personCache, debug,
  1651. __version__, httpPrefix,
  1652. domain, onionDomain)
  1653. if pubKey:
  1654. if debug:
  1655. print('DEBUG: public key obtained for reply: ' + lookupActor)
  1656. break
  1657. if debug:
  1658. print('DEBUG: Retry ' + str(tries + 1) +
  1659. ' obtaining actor for ' + lookupActor)
  1660. time.sleep(5)
  1661. def _dmNotify(baseDir: str, handle: str, url: str) -> None:
  1662. """Creates a notification that a new DM has arrived
  1663. """
  1664. accountDir = baseDir + '/accounts/' + handle
  1665. if not os.path.isdir(accountDir):
  1666. return
  1667. dmFile = accountDir + '/.newDM'
  1668. if not os.path.isfile(dmFile):
  1669. with open(dmFile, 'w+') as fp:
  1670. fp.write(url)
  1671. def _alreadyLiked(baseDir: str, nickname: str, domain: str,
  1672. postUrl: str, likerActor: str) -> bool:
  1673. """Is the given post already liked by the given handle?
  1674. """
  1675. postFilename = \
  1676. locatePost(baseDir, nickname, domain, postUrl)
  1677. if not postFilename:
  1678. return False
  1679. postJsonObject = loadJson(postFilename, 1)
  1680. if not postJsonObject:
  1681. return False
  1682. if not postJsonObject.get('object'):
  1683. return False
  1684. if not isinstance(postJsonObject['object'], dict):
  1685. return False
  1686. if not postJsonObject['object'].get('likes'):
  1687. return False
  1688. if not postJsonObject['object']['likes'].get('items'):
  1689. return False
  1690. for like in postJsonObject['object']['likes']['items']:
  1691. if not like.get('type'):
  1692. continue
  1693. if not like.get('actor'):
  1694. continue
  1695. if like['type'] != 'Like':
  1696. continue
  1697. if like['actor'] == likerActor:
  1698. return True
  1699. return False
  1700. def _likeNotify(baseDir: str, domain: str, onionDomain: str,
  1701. handle: str, actor: str, url: str) -> None:
  1702. """Creates a notification that a like has arrived
  1703. """
  1704. # This is not you liking your own post
  1705. if actor in url:
  1706. return
  1707. # check that the liked post was by this handle
  1708. nickname = handle.split('@')[0]
  1709. if '/' + domain + '/users/' + nickname not in url:
  1710. if not onionDomain:
  1711. return
  1712. if '/' + onionDomain + '/users/' + nickname not in url:
  1713. return
  1714. accountDir = baseDir + '/accounts/' + handle
  1715. # are like notifications enabled?
  1716. notifyLikesEnabledFilename = accountDir + '/.notifyLikes'
  1717. if not os.path.isfile(notifyLikesEnabledFilename):
  1718. return
  1719. likeFile = accountDir + '/.newLike'
  1720. if os.path.isfile(likeFile):
  1721. if '##sent##' not in open(likeFile).read():
  1722. return
  1723. likerNickname = getNicknameFromActor(actor)
  1724. likerDomain, likerPort = getDomainFromActor(actor)
  1725. if likerNickname and likerDomain:
  1726. likerHandle = likerNickname + '@' + likerDomain
  1727. else:
  1728. print('_likeNotify likerHandle: ' +
  1729. str(likerNickname) + '@' + str(likerDomain))
  1730. likerHandle = actor
  1731. if likerHandle != handle:
  1732. likeStr = likerHandle + ' ' + url + '?likedBy=' + actor
  1733. prevLikeFile = accountDir + '/.prevLike'
  1734. # was there a previous like notification?
  1735. if os.path.isfile(prevLikeFile):
  1736. # is it the same as the current notification ?
  1737. with open(prevLikeFile, 'r') as fp:
  1738. prevLikeStr = fp.read()
  1739. if prevLikeStr == likeStr:
  1740. return
  1741. try:
  1742. with open(prevLikeFile, 'w+') as fp:
  1743. fp.write(likeStr)
  1744. except BaseException:
  1745. print('ERROR: unable to save previous like notification ' +
  1746. prevLikeFile)
  1747. pass
  1748. try:
  1749. with open(likeFile, 'w+') as fp:
  1750. fp.write(likeStr)
  1751. except BaseException:
  1752. print('ERROR: unable to write like notification file ' +
  1753. likeFile)
  1754. pass
  1755. def _replyNotify(baseDir: str, handle: str, url: str) -> None:
  1756. """Creates a notification that a new reply has arrived
  1757. """
  1758. accountDir = baseDir + '/accounts/' + handle
  1759. if not os.path.isdir(accountDir):
  1760. return
  1761. replyFile = accountDir + '/.newReply'
  1762. if not os.path.isfile(replyFile):
  1763. with open(replyFile, 'w+') as fp:
  1764. fp.write(url)
  1765. def _gitPatchNotify(baseDir: str, handle: str,
  1766. subject: str, content: str,
  1767. fromNickname: str, fromDomain: str) -> None:
  1768. """Creates a notification that a new git patch has arrived
  1769. """
  1770. accountDir = baseDir + '/accounts/' + handle
  1771. if not os.path.isdir(accountDir):
  1772. return
  1773. patchFile = accountDir + '/.newPatch'
  1774. subject = subject.replace('[PATCH]', '').strip()
  1775. handle = '@' + fromNickname + '@' + fromDomain
  1776. with open(patchFile, 'w+') as fp:
  1777. fp.write('git ' + handle + ' ' + subject)
  1778. def _groupHandle(baseDir: str, handle: str) -> bool:
  1779. """Is the given account handle a group?
  1780. """
  1781. actorFile = baseDir + '/accounts/' + handle + '.json'
  1782. if not os.path.isfile(actorFile):
  1783. return False
  1784. actorJson = loadJson(actorFile)
  1785. if not actorJson:
  1786. return False
  1787. return actorJson['type'] == 'Group'
  1788. def _getGroupName(baseDir: str, handle: str) -> str:
  1789. """Returns the preferred name of a group
  1790. """
  1791. actorFile = baseDir + '/accounts/' + handle + '.json'
  1792. if not os.path.isfile(actorFile):
  1793. return False
  1794. actorJson = loadJson(actorFile)
  1795. if not actorJson:
  1796. return 'Group'
  1797. return actorJson['name']
  1798. def _sendToGroupMembers(session, baseDir: str, handle: str, port: int,
  1799. postJsonObject: {},
  1800. httpPrefix: str, federationList: [],
  1801. sendThreads: [], postLog: [], cachedWebfingers: {},
  1802. personCache: {}, debug: bool) -> None:
  1803. """When a post arrives for a group send it out to the group members
  1804. """
  1805. followersFile = baseDir + '/accounts/' + handle + '/followers.txt'
  1806. if not os.path.isfile(followersFile):
  1807. return
  1808. if not postJsonObject.get('object'):
  1809. return
  1810. nickname = handle.split('@')[0]
  1811. # groupname = _getGroupName(baseDir, handle)
  1812. domain = handle.split('@')[1]
  1813. domainFull = getFullDomain(domain, port)
  1814. # set sender
  1815. cc = ''
  1816. sendingActor = postJsonObject['actor']
  1817. sendingActorNickname = getNicknameFromActor(sendingActor)
  1818. sendingActorDomain, sendingActorPort = \
  1819. getDomainFromActor(sendingActor)
  1820. sendingActorDomainFull = \
  1821. getFullDomain(sendingActorDomain, sendingActorPort)
  1822. senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull
  1823. if not postJsonObject['object']['content'].startswith(senderStr):
  1824. postJsonObject['object']['content'] = \
  1825. senderStr + ' ' + postJsonObject['object']['content']
  1826. # add mention to tag list
  1827. if not postJsonObject['object']['tag']:
  1828. postJsonObject['object']['tag'] = []
  1829. # check if the mention already exists
  1830. mentionExists = False
  1831. for mention in postJsonObject['object']['tag']:
  1832. if mention['type'] == 'Mention':
  1833. if mention.get('href'):
  1834. if mention['href'] == sendingActor:
  1835. mentionExists = True
  1836. if not mentionExists:
  1837. # add the mention of the original sender
  1838. postJsonObject['object']['tag'].append({
  1839. 'href': sendingActor,
  1840. 'name': senderStr,
  1841. 'type': 'Mention'
  1842. })
  1843. postJsonObject['actor'] = \
  1844. httpPrefix + '://' + domainFull + '/users/' + nickname
  1845. postJsonObject['to'] = \
  1846. [postJsonObject['actor'] + '/followers']
  1847. postJsonObject['cc'] = [cc]
  1848. postJsonObject['object']['to'] = postJsonObject['to']
  1849. postJsonObject['object']['cc'] = [cc]
  1850. # set subject
  1851. if not postJsonObject['object'].get('summary'):
  1852. postJsonObject['object']['summary'] = 'General Discussion'
  1853. if ':' in domain:
  1854. domain = domain.split(':')[0]
  1855. with open(followersFile, 'r') as groupMembers:
  1856. for memberHandle in groupMembers:
  1857. if memberHandle != handle:
  1858. memberNickname = memberHandle.split('@')[0]
  1859. memberDomain = memberHandle.split('@')[1]
  1860. memberPort = port
  1861. if ':' in memberDomain:
  1862. memberPortStr = memberDomain.split(':')[1]
  1863. if memberPortStr.isdigit():
  1864. memberPort = int(memberPortStr)
  1865. memberDomain = memberDomain.split(':')[0]
  1866. sendSignedJson(postJsonObject, session, baseDir,
  1867. nickname, domain, port,
  1868. memberNickname, memberDomain, memberPort, cc,
  1869. httpPrefix, False, False, federationList,
  1870. sendThreads, postLog, cachedWebfingers,
  1871. personCache, debug, __version__)
  1872. def _inboxUpdateCalendar(baseDir: str, handle: str,
  1873. postJsonObject: {}) -> None:
  1874. """Detects whether the tag list on a post contains calendar events
  1875. and if so saves the post id to a file in the calendar directory
  1876. for the account
  1877. """
  1878. if not postJsonObject.get('actor'):
  1879. return
  1880. if not postJsonObject.get('object'):
  1881. return
  1882. if not isinstance(postJsonObject['object'], dict):
  1883. return
  1884. if not postJsonObject['object'].get('tag'):
  1885. return
  1886. if not isinstance(postJsonObject['object']['tag'], list):
  1887. return
  1888. actor = postJsonObject['actor']
  1889. actorNickname = getNicknameFromActor(actor)
  1890. actorDomain, actorPort = getDomainFromActor(actor)
  1891. handleNickname = handle.split('@')[0]
  1892. handleDomain = handle.split('@')[1]
  1893. if not receivingCalendarEvents(baseDir,
  1894. handleNickname, handleDomain,
  1895. actorNickname, actorDomain):
  1896. return
  1897. postId = removeIdEnding(postJsonObject['id']).replace('/', '#')
  1898. # look for events within the tags list
  1899. for tagDict in postJsonObject['object']['tag']:
  1900. if not tagDict.get('type'):
  1901. continue
  1902. if tagDict['type'] != 'Event':
  1903. continue
  1904. if not tagDict.get('startTime'):
  1905. continue
  1906. saveEventPost(baseDir, handle, postId, tagDict)
  1907. def inboxUpdateIndex(boxname: str, baseDir: str, handle: str,
  1908. destinationFilename: str, debug: bool) -> bool:
  1909. """Updates the index of received posts
  1910. The new entry is added to the top of the file
  1911. """
  1912. indexFilename = baseDir + '/accounts/' + handle + '/' + boxname + '.index'
  1913. if debug:
  1914. print('DEBUG: Updating index ' + indexFilename)
  1915. if '/' + boxname + '/' in destinationFilename:
  1916. destinationFilename = destinationFilename.split('/' + boxname + '/')[1]
  1917. # remove the path
  1918. if '/' in destinationFilename:
  1919. destinationFilename = destinationFilename.split('/')[-1]
  1920. if os.path.isfile(indexFilename):
  1921. try:
  1922. with open(indexFilename, 'r+') as indexFile:
  1923. content = indexFile.read()
  1924. if destinationFilename + '\n' not in content:
  1925. indexFile.seek(0, 0)
  1926. indexFile.write(destinationFilename + '\n' + content)
  1927. return True
  1928. except Exception as e:
  1929. print('WARN: Failed to write entry to index ' + str(e))
  1930. else:
  1931. try:
  1932. indexFile = open(indexFilename, 'w+')
  1933. if indexFile:
  1934. indexFile.write(destinationFilename + '\n')
  1935. indexFile.close()
  1936. except Exception as e:
  1937. print('WARN: Failed to write initial entry to index ' + str(e))
  1938. return False
  1939. def _updateLastSeen(baseDir: str, handle: str, actor: str) -> None:
  1940. """Updates the time when the given handle last saw the given actor
  1941. This can later be used to indicate if accounts are dormant/abandoned/moved
  1942. """
  1943. if '@' not in handle:
  1944. return
  1945. nickname = handle.split('@')[0]
  1946. domain = handle.split('@')[1]
  1947. if ':' in domain:
  1948. domain = domain.split(':')[0]
  1949. accountPath = baseDir + '/accounts/' + nickname + '@' + domain
  1950. if not os.path.isdir(accountPath):
  1951. return
  1952. if not isFollowingActor(baseDir, nickname, domain, actor):
  1953. return
  1954. lastSeenPath = accountPath + '/lastseen'
  1955. if not os.path.isdir(lastSeenPath):
  1956. os.mkdir(lastSeenPath)
  1957. lastSeenFilename = lastSeenPath + '/' + actor.replace('/', '#') + '.txt'
  1958. currTime = datetime.datetime.utcnow()
  1959. daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days
  1960. # has the value changed?
  1961. if os.path.isfile(lastSeenFilename):
  1962. with open(lastSeenFilename, 'r') as lastSeenFile:
  1963. daysSinceEpochFile = lastSeenFile.read()
  1964. if int(daysSinceEpochFile) == daysSinceEpoch:
  1965. # value hasn't changed, so we can save writing anything to file
  1966. return
  1967. with open(lastSeenFilename, 'w+') as lastSeenFile:
  1968. lastSeenFile.write(str(daysSinceEpoch))
  1969. def _bounceDM(senderPostId: str, session, httpPrefix: str,
  1970. baseDir: str, nickname: str, domain: str, port: int,
  1971. sendingHandle: str, federationList: [],
  1972. sendThreads: [], postLog: [],
  1973. cachedWebfingers: {}, personCache: {},
  1974. translate: {}, debug: bool,
  1975. lastBounceMessage: []) -> bool:
  1976. """Sends a bounce message back to the sending handle
  1977. if a DM has been rejected
  1978. """
  1979. print(nickname + '@' + domain +
  1980. ' cannot receive DM from ' + sendingHandle +
  1981. ' because they do not follow them')
  1982. # Don't send out bounce messages too frequently.
  1983. # Otherwise an adversary could try to DoS your instance
  1984. # by continuously sending DMs to you
  1985. currTime = int(time.time())
  1986. if currTime - lastBounceMessage[0] < 60:
  1987. return False
  1988. # record the last time that a bounce was generated
  1989. lastBounceMessage[0] = currTime
  1990. senderNickname = sendingHandle.split('@')[0]
  1991. senderDomain = sendingHandle.split('@')[1]
  1992. senderPort = port
  1993. if ':' in senderDomain:
  1994. senderPortStr = senderDomain.split(':')[1]
  1995. if senderPortStr.isdigit():
  1996. senderPort = int(senderPortStr)
  1997. senderDomain = senderDomain.split(':')[0]
  1998. cc = []
  1999. # create the bounce DM
  2000. subject = None
  2001. content = translate['DM bounce']
  2002. followersOnly = False
  2003. saveToFile = False
  2004. clientToServer = False
  2005. commentsEnabled = False
  2006. attachImageFilename = None
  2007. mediaType = None
  2008. imageDescription = ''
  2009. city = 'London, England'
  2010. inReplyTo = removeIdEnding(senderPostId)
  2011. inReplyToAtomUri = None
  2012. schedulePost = False
  2013. eventDate = None
  2014. eventTime = None
  2015. location = None
  2016. postJsonObject = \
  2017. createDirectMessagePost(baseDir, nickname, domain, port,
  2018. httpPrefix, content, followersOnly,
  2019. saveToFile, clientToServer,
  2020. commentsEnabled,
  2021. attachImageFilename, mediaType,
  2022. imageDescription, city,
  2023. inReplyTo, inReplyToAtomUri,
  2024. subject, debug, schedulePost,
  2025. eventDate, eventTime, location)
  2026. if not postJsonObject:
  2027. print('WARN: unable to create bounce message to ' + sendingHandle)
  2028. return False
  2029. # bounce DM goes back to the sender
  2030. print('Sending bounce DM to ' + sendingHandle)
  2031. sendSignedJson(postJsonObject, session, baseDir,
  2032. nickname, domain, port,
  2033. senderNickname, senderDomain, senderPort, cc,
  2034. httpPrefix, False, False, federationList,
  2035. sendThreads, postLog, cachedWebfingers,
  2036. personCache, debug, __version__)
  2037. return True
  2038. def _inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
  2039. session, keyId: str, handle: str, messageJson: {},
  2040. baseDir: str, httpPrefix: str, sendThreads: [],
  2041. postLog: [], cachedWebfingers: {}, personCache: {},
  2042. queue: [], domain: str,
  2043. onionDomain: str, i2pDomain: str,
  2044. port: int, proxyType: str,
  2045. federationList: [], debug: bool,
  2046. queueFilename: str, destinationFilename: str,
  2047. maxReplies: int, allowDeletion: bool,
  2048. maxMentions: int, maxEmoji: int, translate: {},
  2049. unitTest: bool, YTReplacementDomain: str,
  2050. showPublishedDateOnly: bool,
  2051. allowLocalNetworkAccess: bool,
  2052. peertubeInstances: [],
  2053. lastBounceMessage: [],
  2054. themeName: str) -> bool:
  2055. """ Anything which needs to be done after initial checks have passed
  2056. """
  2057. actor = keyId
  2058. if '#' in actor:
  2059. actor = keyId.split('#')[0]
  2060. _updateLastSeen(baseDir, handle, actor)
  2061. isGroup = _groupHandle(baseDir, handle)
  2062. if _receiveLike(recentPostsCache,
  2063. session, handle, isGroup,
  2064. baseDir, httpPrefix,
  2065. domain, port,
  2066. onionDomain,
  2067. sendThreads, postLog,
  2068. cachedWebfingers,
  2069. personCache,
  2070. messageJson,
  2071. federationList,
  2072. debug):
  2073. if debug:
  2074. print('DEBUG: Like accepted from ' + actor)
  2075. return False
  2076. if _receiveUndoLike(recentPostsCache,
  2077. session, handle, isGroup,
  2078. baseDir, httpPrefix,
  2079. domain, port,
  2080. sendThreads, postLog,
  2081. cachedWebfingers,
  2082. personCache,
  2083. messageJson,
  2084. federationList,
  2085. debug):
  2086. if debug:
  2087. print('DEBUG: Undo like accepted from ' + actor)
  2088. return False
  2089. if _receiveBookmark(recentPostsCache,
  2090. session, handle, isGroup,
  2091. baseDir, httpPrefix,
  2092. domain, port,
  2093. sendThreads, postLog,
  2094. cachedWebfingers,
  2095. personCache,
  2096. messageJson,
  2097. federationList,
  2098. debug):
  2099. if debug:
  2100. print('DEBUG: Bookmark accepted from ' + actor)
  2101. return False
  2102. if _receiveUndoBookmark(recentPostsCache,
  2103. session, handle, isGroup,
  2104. baseDir, httpPrefix,
  2105. domain, port,
  2106. sendThreads, postLog,
  2107. cachedWebfingers,
  2108. personCache,
  2109. messageJson,
  2110. federationList,
  2111. debug):
  2112. if debug:
  2113. print('DEBUG: Undo bookmark accepted from ' + actor)
  2114. return False
  2115. if _receiveAnnounce(recentPostsCache,
  2116. session, handle, isGroup,
  2117. baseDir, httpPrefix,
  2118. domain, onionDomain, port,
  2119. sendThreads, postLog,
  2120. cachedWebfingers,
  2121. personCache,
  2122. messageJson,
  2123. federationList,
  2124. debug, translate,
  2125. YTReplacementDomain,
  2126. allowLocalNetworkAccess,
  2127. themeName):
  2128. if debug:
  2129. print('DEBUG: Announce accepted from ' + actor)
  2130. if _receiveUndoAnnounce(recentPostsCache,
  2131. session, handle, isGroup,
  2132. baseDir, httpPrefix,
  2133. domain, port,
  2134. sendThreads, postLog,
  2135. cachedWebfingers,
  2136. personCache,
  2137. messageJson,
  2138. federationList,
  2139. debug):
  2140. if debug:
  2141. print('DEBUG: Undo announce accepted from ' + actor)
  2142. return False
  2143. if _receiveDelete(session, handle, isGroup,
  2144. baseDir, httpPrefix,
  2145. domain, port,
  2146. sendThreads, postLog,
  2147. cachedWebfingers,
  2148. personCache,
  2149. messageJson,
  2150. federationList,
  2151. debug, allowDeletion,
  2152. recentPostsCache):
  2153. if debug:
  2154. print('DEBUG: Delete accepted from ' + actor)
  2155. return False
  2156. if debug:
  2157. print('DEBUG: initial checks passed')
  2158. print('copy queue file from ' + queueFilename +
  2159. ' to ' + destinationFilename)
  2160. if os.path.isfile(destinationFilename):
  2161. return True
  2162. if messageJson.get('postNickname'):
  2163. postJsonObject = messageJson['post']
  2164. else:
  2165. postJsonObject = messageJson
  2166. nickname = handle.split('@')[0]
  2167. if _validPostContent(baseDir, nickname, domain,
  2168. postJsonObject, maxMentions, maxEmoji,
  2169. allowLocalNetworkAccess, debug):
  2170. if postJsonObject.get('object'):
  2171. jsonObj = postJsonObject['object']
  2172. if not isinstance(jsonObj, dict):
  2173. jsonObj = None
  2174. else:
  2175. jsonObj = postJsonObject
  2176. # check for incoming git patches
  2177. if jsonObj:
  2178. if jsonObj.get('content') and \
  2179. jsonObj.get('summary') and \
  2180. jsonObj.get('attributedTo'):
  2181. attributedTo = jsonObj['attributedTo']
  2182. if isinstance(attributedTo, str):
  2183. fromNickname = getNicknameFromActor(attributedTo)
  2184. fromDomain, fromPort = getDomainFromActor(attributedTo)
  2185. fromDomain = getFullDomain(fromDomain, fromPort)
  2186. if receiveGitPatch(baseDir, nickname, domain,
  2187. jsonObj['type'],
  2188. jsonObj['summary'],
  2189. jsonObj['content'],
  2190. fromNickname, fromDomain):
  2191. _gitPatchNotify(baseDir, handle,
  2192. jsonObj['summary'],
  2193. jsonObj['content'],
  2194. fromNickname, fromDomain)
  2195. elif '[PATCH]' in jsonObj['content']:
  2196. print('WARN: git patch not accepted - ' +
  2197. jsonObj['summary'])
  2198. return False
  2199. # replace YouTube links, so they get less tracking data
  2200. replaceYouTube(postJsonObject, YTReplacementDomain)
  2201. # list of indexes to be updated
  2202. updateIndexList = ['inbox']
  2203. populateReplies(baseDir, httpPrefix, domain, postJsonObject,
  2204. maxReplies, debug)
  2205. # if this is a reply to a question then update the votes
  2206. questionJson = questionUpdateVotes(baseDir, nickname, domain,
  2207. postJsonObject)
  2208. if questionJson:
  2209. # Is this a question created by this instance?
  2210. idPrefix = httpPrefix + '://' + domain
  2211. if questionJson['object']['id'].startswith(idPrefix):
  2212. # if the votes on a question have changed then
  2213. # send out an update
  2214. questionJson['type'] = 'Update'
  2215. sendToFollowersThread(session, baseDir,
  2216. nickname, domain,
  2217. onionDomain, i2pDomain, port,
  2218. httpPrefix, federationList,
  2219. sendThreads, postLog,
  2220. cachedWebfingers, personCache,
  2221. postJsonObject, debug,
  2222. __version__)
  2223. isReplyToMutedPost = False
  2224. if not isGroup:
  2225. # create a DM notification file if needed
  2226. postIsDM = isDM(postJsonObject)
  2227. if postIsDM:
  2228. if nickname != 'inbox':
  2229. # check for the flag file which indicates to
  2230. # only receive DMs from people you are following
  2231. followDMsFilename = \
  2232. baseDir + '/accounts/' + \
  2233. nickname + '@' + domain + '/.followDMs'
  2234. if os.path.isfile(followDMsFilename):
  2235. # get the file containing following handles
  2236. followingFilename = \
  2237. baseDir + '/accounts/' + \
  2238. nickname + '@' + domain + '/following.txt'
  2239. # who is sending a DM?
  2240. if not postJsonObject.get('actor'):
  2241. return False
  2242. sendingActor = postJsonObject['actor']
  2243. sendingActorNickname = \
  2244. getNicknameFromActor(sendingActor)
  2245. if not sendingActorNickname:
  2246. return False
  2247. sendingActorDomain, sendingActorPort = \
  2248. getDomainFromActor(sendingActor)
  2249. if not sendingActorDomain:
  2250. return False
  2251. sendingToSelf = False
  2252. if sendingActorNickname == nickname and \
  2253. sendingActorDomain == domain:
  2254. sendingToSelf = True
  2255. # check that the following file exists
  2256. if not sendingToSelf:
  2257. if not os.path.isfile(followingFilename):
  2258. print('No following.txt file exists for ' +
  2259. nickname + '@' + domain +
  2260. ' so not accepting DM from ' +
  2261. sendingActorNickname + '@' +
  2262. sendingActorDomain)
  2263. return False
  2264. # Not sending to yourself
  2265. if not sendingToSelf:
  2266. # get the handle of the DM sender
  2267. sendH = \
  2268. sendingActorNickname + '@' + sendingActorDomain
  2269. # check the follow
  2270. if not isFollowingActor(baseDir,
  2271. nickname, domain,
  2272. sendH):
  2273. # DMs may always be allowed from some domains
  2274. if not dmAllowedFromDomain(baseDir,
  2275. nickname, domain,
  2276. sendingActorDomain):
  2277. # send back a bounce DM
  2278. if postJsonObject.get('id') and \
  2279. postJsonObject.get('object'):
  2280. # don't send bounces back to
  2281. # replies to bounce messages
  2282. obj = postJsonObject['object']
  2283. if isinstance(obj, dict):
  2284. if not obj.get('inReplyTo'):
  2285. senderPostId = \
  2286. postJsonObject['id']
  2287. _bounceDM(senderPostId,
  2288. session, httpPrefix,
  2289. baseDir,
  2290. nickname, domain,
  2291. port, sendH,
  2292. federationList,
  2293. sendThreads, postLog,
  2294. cachedWebfingers,
  2295. personCache,
  2296. translate, debug,
  2297. lastBounceMessage)
  2298. return False
  2299. # dm index will be updated
  2300. updateIndexList.append('dm')
  2301. _dmNotify(baseDir, handle,
  2302. httpPrefix + '://' + domain + '/users/' +
  2303. nickname + '/dm')
  2304. # get the actor being replied to
  2305. domainFull = getFullDomain(domain, port)
  2306. actor = httpPrefix + '://' + domainFull + \
  2307. '/users/' + handle.split('@')[0]
  2308. # create a reply notification file if needed
  2309. if not postIsDM and isReply(postJsonObject, actor):
  2310. if nickname != 'inbox':
  2311. # replies index will be updated
  2312. updateIndexList.append('tlreplies')
  2313. if postJsonObject['object'].get('inReplyTo'):
  2314. inReplyTo = postJsonObject['object']['inReplyTo']
  2315. if inReplyTo:
  2316. if isinstance(inReplyTo, str):
  2317. if not isMuted(baseDir, nickname, domain,
  2318. inReplyTo):
  2319. _replyNotify(baseDir, handle,
  2320. httpPrefix + '://' + domain +
  2321. '/users/' + nickname +
  2322. '/tlreplies')
  2323. else:
  2324. isReplyToMutedPost = True
  2325. if isImageMedia(session, baseDir, httpPrefix,
  2326. nickname, domain, postJsonObject,
  2327. translate, YTReplacementDomain,
  2328. allowLocalNetworkAccess,
  2329. recentPostsCache, debug):
  2330. # media index will be updated
  2331. updateIndexList.append('tlmedia')
  2332. if isBlogPost(postJsonObject):
  2333. # blogs index will be updated
  2334. updateIndexList.append('tlblogs')
  2335. elif isEventPost(postJsonObject):
  2336. # events index will be updated
  2337. updateIndexList.append('tlevents')
  2338. # get the avatar for a reply/announce
  2339. _obtainAvatarForReplyPost(session, baseDir,
  2340. httpPrefix, domain, onionDomain,
  2341. personCache, postJsonObject, debug)
  2342. # save the post to file
  2343. if saveJson(postJsonObject, destinationFilename):
  2344. # If this is a reply to a muted post then also mute it.
  2345. # This enables you to ignore a threat that's getting boring
  2346. if isReplyToMutedPost:
  2347. print('MUTE REPLY: ' + destinationFilename)
  2348. muteFile = open(destinationFilename + '.muted', 'w+')
  2349. if muteFile:
  2350. muteFile.write('\n')
  2351. muteFile.close()
  2352. # update the indexes for different timelines
  2353. for boxname in updateIndexList:
  2354. if not inboxUpdateIndex(boxname, baseDir, handle,
  2355. destinationFilename, debug):
  2356. print('ERROR: unable to update ' + boxname + ' index')
  2357. else:
  2358. if boxname == 'inbox':
  2359. if isRecentPost(postJsonObject):
  2360. domainFull = getFullDomain(domain, port)
  2361. updateSpeaker(baseDir, httpPrefix,
  2362. nickname, domain, domainFull,
  2363. postJsonObject, personCache,
  2364. translate, None, themeName)
  2365. if not unitTest:
  2366. if debug:
  2367. print('Saving inbox post as html to cache')
  2368. htmlCacheStartTime = time.time()
  2369. handleName = handle.split('@')[0]
  2370. _inboxStorePostToHtmlCache(recentPostsCache,
  2371. maxRecentPosts,
  2372. translate, baseDir,
  2373. httpPrefix,
  2374. session, cachedWebfingers,
  2375. personCache,
  2376. handleName,
  2377. domain, port,
  2378. postJsonObject,
  2379. allowDeletion,
  2380. boxname,
  2381. showPublishedDateOnly,
  2382. peertubeInstances,
  2383. allowLocalNetworkAccess,
  2384. themeName)
  2385. if debug:
  2386. timeDiff = \
  2387. str(int((time.time() - htmlCacheStartTime) *
  2388. 1000))
  2389. print('Saved ' + boxname +
  2390. ' post as html to cache in ' +
  2391. timeDiff + ' mS')
  2392. _inboxUpdateCalendar(baseDir, handle, postJsonObject)
  2393. handleName = handle.split('@')[0]
  2394. storeHashTags(baseDir, handleName, postJsonObject)
  2395. # send the post out to group members
  2396. if isGroup:
  2397. _sendToGroupMembers(session, baseDir, handle, port,
  2398. postJsonObject,
  2399. httpPrefix, federationList, sendThreads,
  2400. postLog, cachedWebfingers, personCache,
  2401. debug)
  2402. # if the post wasn't saved
  2403. if not os.path.isfile(destinationFilename):
  2404. return False
  2405. return True
  2406. def clearQueueItems(baseDir: str, queue: []) -> None:
  2407. """Clears the queue for each account
  2408. """
  2409. ctr = 0
  2410. queue.clear()
  2411. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  2412. for account in dirs:
  2413. queueDir = baseDir + '/accounts/' + account + '/queue'
  2414. if not os.path.isdir(queueDir):
  2415. continue
  2416. for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
  2417. for qfile in queuefiles:
  2418. try:
  2419. os.remove(os.path.join(queueDir, qfile))
  2420. ctr += 1
  2421. except BaseException:
  2422. pass
  2423. break
  2424. if ctr > 0:
  2425. print('Removed ' + str(ctr) + ' inbox queue items')
  2426. def _restoreQueueItems(baseDir: str, queue: []) -> None:
  2427. """Checks the queue for each account and appends filenames
  2428. """
  2429. queue.clear()
  2430. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  2431. for account in dirs:
  2432. queueDir = baseDir + '/accounts/' + account + '/queue'
  2433. if not os.path.isdir(queueDir):
  2434. continue
  2435. for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
  2436. for qfile in queuefiles:
  2437. queue.append(os.path.join(queueDir, qfile))
  2438. break
  2439. if len(queue) > 0:
  2440. print('Restored ' + str(len(queue)) + ' inbox queue items')
  2441. def runInboxQueueWatchdog(projectVersion: str, httpd) -> None:
  2442. """This tries to keep the inbox thread running even if it dies
  2443. """
  2444. print('Starting inbox queue watchdog')
  2445. inboxQueueOriginal = httpd.thrInboxQueue.clone(runInboxQueue)
  2446. httpd.thrInboxQueue.start()
  2447. while True:
  2448. time.sleep(20)
  2449. if not httpd.thrInboxQueue.is_alive() or httpd.restartInboxQueue:
  2450. httpd.restartInboxQueueInProgress = True
  2451. httpd.thrInboxQueue.kill()
  2452. httpd.thrInboxQueue = inboxQueueOriginal.clone(runInboxQueue)
  2453. httpd.inboxQueue.clear()
  2454. httpd.thrInboxQueue.start()
  2455. print('Restarting inbox queue...')
  2456. httpd.restartInboxQueueInProgress = False
  2457. httpd.restartInboxQueue = False
  2458. def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
  2459. projectVersion: str,
  2460. baseDir: str, httpPrefix: str, sendThreads: [], postLog: [],
  2461. cachedWebfingers: {}, personCache: {}, queue: [],
  2462. domain: str,
  2463. onionDomain: str, i2pDomain: str, port: int, proxyType: str,
  2464. federationList: [], maxReplies: int,
  2465. domainMaxPostsPerDay: int, accountMaxPostsPerDay: int,
  2466. allowDeletion: bool, debug: bool, maxMentions: int,
  2467. maxEmoji: int, translate: {}, unitTest: bool,
  2468. YTReplacementDomain: str,
  2469. showPublishedDateOnly: bool,
  2470. maxFollowers: int, allowLocalNetworkAccess: bool,
  2471. peertubeInstances: [],
  2472. verifyAllSignatures: bool,
  2473. themeName: str) -> None:
  2474. """Processes received items and moves them to the appropriate
  2475. directories
  2476. """
  2477. currSessionTime = int(time.time())
  2478. sessionLastUpdate = currSessionTime
  2479. print('Starting new session when starting inbox queue')
  2480. session = createSession(proxyType)
  2481. inboxHandle = 'inbox@' + domain
  2482. if debug:
  2483. print('DEBUG: Inbox queue running')
  2484. # if queue processing was interrupted (eg server crash)
  2485. # then this loads any outstanding items back into the queue
  2486. _restoreQueueItems(baseDir, queue)
  2487. # keep track of numbers of incoming posts per day
  2488. quotasLastUpdateDaily = int(time.time())
  2489. quotasDaily = {
  2490. 'domains': {},
  2491. 'accounts': {}
  2492. }
  2493. quotasLastUpdatePerMin = int(time.time())
  2494. quotasPerMin = {
  2495. 'domains': {},
  2496. 'accounts': {}
  2497. }
  2498. heartBeatCtr = 0
  2499. queueRestoreCtr = 0
  2500. # time when the last DM bounce message was sent
  2501. # This is in a list so that it can be changed by reference
  2502. # within _bounceDM
  2503. lastBounceMessage = [int(time.time())]
  2504. while True:
  2505. time.sleep(1)
  2506. # heartbeat to monitor whether the inbox queue is running
  2507. heartBeatCtr += 1
  2508. if heartBeatCtr >= 10:
  2509. # turn off broch mode after it has timed out
  2510. brochModeLapses(baseDir)
  2511. print('>>> Heartbeat Q:' + str(len(queue)) + ' ' +
  2512. '{:%F %T}'.format(datetime.datetime.now()))
  2513. heartBeatCtr = 0
  2514. if len(queue) == 0:
  2515. # restore any remaining queue items
  2516. queueRestoreCtr += 1
  2517. if queueRestoreCtr >= 30:
  2518. queueRestoreCtr = 0
  2519. _restoreQueueItems(baseDir, queue)
  2520. continue
  2521. currTime = int(time.time())
  2522. # recreate the session periodically
  2523. if not session or currTime - sessionLastUpdate > 21600:
  2524. print('Regenerating inbox queue session at 6hr interval')
  2525. session = createSession(proxyType)
  2526. if not session:
  2527. continue
  2528. sessionLastUpdate = currTime
  2529. # oldest item first
  2530. queue.sort()
  2531. queueFilename = queue[0]
  2532. if not os.path.isfile(queueFilename):
  2533. print("Queue: queue item rejected because it has no file: " +
  2534. queueFilename)
  2535. if len(queue) > 0:
  2536. queue.pop(0)
  2537. continue
  2538. if debug:
  2539. print('Loading queue item ' + queueFilename)
  2540. # Load the queue json
  2541. queueJson = loadJson(queueFilename, 1)
  2542. if not queueJson:
  2543. print('Queue: runInboxQueue failed to load inbox queue item ' +
  2544. queueFilename)
  2545. # Assume that the file is probably corrupt/unreadable
  2546. if len(queue) > 0:
  2547. queue.pop(0)
  2548. # delete the queue file
  2549. if os.path.isfile(queueFilename):
  2550. try:
  2551. os.remove(queueFilename)
  2552. except BaseException:
  2553. pass
  2554. continue
  2555. # clear the daily quotas for maximum numbers of received posts
  2556. if currTime - quotasLastUpdateDaily > 60 * 60 * 24:
  2557. quotasDaily = {
  2558. 'domains': {},
  2559. 'accounts': {}
  2560. }
  2561. quotasLastUpdateDaily = currTime
  2562. if currTime - quotasLastUpdatePerMin > 60:
  2563. # clear the per minute quotas for maximum numbers of received posts
  2564. quotasPerMin = {
  2565. 'domains': {},
  2566. 'accounts': {}
  2567. }
  2568. # also check if the json signature enforcement has changed
  2569. verifyAllSigs = getConfigParam(baseDir, "verifyAllSignatures")
  2570. if verifyAllSigs is not None:
  2571. verifyAllSignatures = verifyAllSigs
  2572. # change the last time that this was done
  2573. quotasLastUpdatePerMin = currTime
  2574. # limit the number of posts which can arrive per domain per day
  2575. postDomain = queueJson['postDomain']
  2576. if postDomain:
  2577. if domainMaxPostsPerDay > 0:
  2578. if quotasDaily['domains'].get(postDomain):
  2579. if quotasDaily['domains'][postDomain] > \
  2580. domainMaxPostsPerDay:
  2581. print('Queue: Quota per day - Maximum posts for ' +
  2582. postDomain + ' reached (' +
  2583. str(domainMaxPostsPerDay) + ')')
  2584. if len(queue) > 0:
  2585. try:
  2586. os.remove(queueFilename)
  2587. except BaseException:
  2588. pass
  2589. queue.pop(0)
  2590. continue
  2591. quotasDaily['domains'][postDomain] += 1
  2592. else:
  2593. quotasDaily['domains'][postDomain] = 1
  2594. if quotasPerMin['domains'].get(postDomain):
  2595. domainMaxPostsPerMin = \
  2596. int(domainMaxPostsPerDay / (24 * 60))
  2597. if domainMaxPostsPerMin < 5:
  2598. domainMaxPostsPerMin = 5
  2599. if quotasPerMin['domains'][postDomain] > \
  2600. domainMaxPostsPerMin:
  2601. print('Queue: Quota per min - Maximum posts for ' +
  2602. postDomain + ' reached (' +
  2603. str(domainMaxPostsPerMin) + ')')
  2604. if len(queue) > 0:
  2605. try:
  2606. os.remove(queueFilename)
  2607. except BaseException:
  2608. pass
  2609. queue.pop(0)
  2610. continue
  2611. quotasPerMin['domains'][postDomain] += 1
  2612. else:
  2613. quotasPerMin['domains'][postDomain] = 1
  2614. if accountMaxPostsPerDay > 0:
  2615. postHandle = queueJson['postNickname'] + '@' + postDomain
  2616. if quotasDaily['accounts'].get(postHandle):
  2617. if quotasDaily['accounts'][postHandle] > \
  2618. accountMaxPostsPerDay:
  2619. print('Queue: Quota account posts per day -' +
  2620. ' Maximum posts for ' +
  2621. postHandle + ' reached (' +
  2622. str(accountMaxPostsPerDay) + ')')
  2623. if len(queue) > 0:
  2624. try:
  2625. os.remove(queueFilename)
  2626. except BaseException:
  2627. pass
  2628. queue.pop(0)
  2629. continue
  2630. quotasDaily['accounts'][postHandle] += 1
  2631. else:
  2632. quotasDaily['accounts'][postHandle] = 1
  2633. if quotasPerMin['accounts'].get(postHandle):
  2634. accountMaxPostsPerMin = \
  2635. int(accountMaxPostsPerDay / (24 * 60))
  2636. if accountMaxPostsPerMin < 5:
  2637. accountMaxPostsPerMin = 5
  2638. if quotasPerMin['accounts'][postHandle] > \
  2639. accountMaxPostsPerMin:
  2640. print('Queue: Quota account posts per min -' +
  2641. ' Maximum posts for ' +
  2642. postHandle + ' reached (' +
  2643. str(accountMaxPostsPerMin) + ')')
  2644. if len(queue) > 0:
  2645. try:
  2646. os.remove(queueFilename)
  2647. except BaseException:
  2648. pass
  2649. queue.pop(0)
  2650. continue
  2651. quotasPerMin['accounts'][postHandle] += 1
  2652. else:
  2653. quotasPerMin['accounts'][postHandle] = 1
  2654. if debug:
  2655. if accountMaxPostsPerDay > 0 or domainMaxPostsPerDay > 0:
  2656. pprint(quotasDaily)
  2657. if debug and queueJson.get('actor'):
  2658. print('Obtaining public key for actor ' + queueJson['actor'])
  2659. # Try a few times to obtain the public key
  2660. pubKey = None
  2661. keyId = None
  2662. for tries in range(8):
  2663. keyId = None
  2664. signatureParams = \
  2665. queueJson['httpHeaders']['signature'].split(',')
  2666. for signatureItem in signatureParams:
  2667. if signatureItem.startswith('keyId='):
  2668. if '"' in signatureItem:
  2669. keyId = signatureItem.split('"')[1]
  2670. break
  2671. if not keyId:
  2672. print('Queue: No keyId in signature: ' +
  2673. queueJson['httpHeaders']['signature'])
  2674. pubKey = None
  2675. break
  2676. pubKey = \
  2677. getPersonPubKey(baseDir, session, keyId,
  2678. personCache, debug,
  2679. projectVersion, httpPrefix,
  2680. domain, onionDomain)
  2681. if pubKey:
  2682. if debug:
  2683. print('DEBUG: public key: ' + str(pubKey))
  2684. break
  2685. if debug:
  2686. print('DEBUG: Retry ' + str(tries+1) +
  2687. ' obtaining public key for ' + keyId)
  2688. time.sleep(1)
  2689. if not pubKey:
  2690. if debug:
  2691. print('Queue: public key could not be obtained from ' + keyId)
  2692. if os.path.isfile(queueFilename):
  2693. os.remove(queueFilename)
  2694. if len(queue) > 0:
  2695. queue.pop(0)
  2696. continue
  2697. # check the http header signature
  2698. if debug:
  2699. print('DEBUG: checking http header signature')
  2700. pprint(queueJson['httpHeaders'])
  2701. postStr = json.dumps(queueJson['post'])
  2702. httpSignatureFailed = False
  2703. if not verifyPostHeaders(httpPrefix,
  2704. pubKey,
  2705. queueJson['httpHeaders'],
  2706. queueJson['path'], False,
  2707. queueJson['digest'],
  2708. postStr,
  2709. debug):
  2710. httpSignatureFailed = True
  2711. print('Queue: Header signature check failed')
  2712. pprint(queueJson['httpHeaders'])
  2713. else:
  2714. if debug:
  2715. print('DEBUG: http header signature check success')
  2716. # check if a json signature exists on this post
  2717. hasJsonSignature = False
  2718. jwebsigType = None
  2719. originalJson = queueJson['original']
  2720. if originalJson.get('@context') and \
  2721. originalJson.get('signature'):
  2722. if isinstance(originalJson['signature'], dict):
  2723. # see https://tools.ietf.org/html/rfc7515
  2724. jwebsig = originalJson['signature']
  2725. # signature exists and is of the expected type
  2726. if jwebsig.get('type') and jwebsig.get('signatureValue'):
  2727. jwebsigType = jwebsig['type']
  2728. if jwebsigType == 'RsaSignature2017':
  2729. if hasValidContext(originalJson):
  2730. hasJsonSignature = True
  2731. else:
  2732. unknownContextsFile = \
  2733. baseDir + '/accounts/unknownContexts.txt'
  2734. unknownContext = str(originalJson['@context'])
  2735. print('unrecognized @context: ' +
  2736. unknownContext)
  2737. alreadyUnknown = False
  2738. if os.path.isfile(unknownContextsFile):
  2739. if unknownContext in \
  2740. open(unknownContextsFile).read():
  2741. alreadyUnknown = True
  2742. if not alreadyUnknown:
  2743. unknownFile = open(unknownContextsFile, "a+")
  2744. if unknownFile:
  2745. unknownFile.write(unknownContext + '\n')
  2746. unknownFile.close()
  2747. else:
  2748. print('Unrecognized jsonld signature type: ' +
  2749. jwebsigType)
  2750. unknownSignaturesFile = \
  2751. baseDir + '/accounts/unknownJsonSignatures.txt'
  2752. alreadyUnknown = False
  2753. if os.path.isfile(unknownSignaturesFile):
  2754. if jwebsigType in \
  2755. open(unknownSignaturesFile).read():
  2756. alreadyUnknown = True
  2757. if not alreadyUnknown:
  2758. unknownFile = open(unknownSignaturesFile, "a+")
  2759. if unknownFile:
  2760. unknownFile.write(jwebsigType + '\n')
  2761. unknownFile.close()
  2762. # strict enforcement of json signatures
  2763. if not hasJsonSignature:
  2764. if httpSignatureFailed:
  2765. if jwebsigType:
  2766. print('Queue: Header signature check failed and does ' +
  2767. 'not have a recognised jsonld signature type ' +
  2768. jwebsigType)
  2769. else:
  2770. print('Queue: Header signature check failed and ' +
  2771. 'does not have jsonld signature')
  2772. if debug:
  2773. pprint(queueJson['httpHeaders'])
  2774. if verifyAllSignatures:
  2775. print('Queue: inbox post does not have a jsonld signature ' +
  2776. keyId + ' ' + str(originalJson))
  2777. if httpSignatureFailed or verifyAllSignatures:
  2778. if os.path.isfile(queueFilename):
  2779. os.remove(queueFilename)
  2780. if len(queue) > 0:
  2781. queue.pop(0)
  2782. continue
  2783. else:
  2784. if httpSignatureFailed or verifyAllSignatures:
  2785. # use the original json message received, not one which
  2786. # may have been modified along the way
  2787. if not verifyJsonSignature(originalJson, pubKey):
  2788. if debug:
  2789. print('WARN: jsonld inbox signature check failed ' +
  2790. keyId + ' ' + pubKey + ' ' + str(originalJson))
  2791. else:
  2792. print('WARN: jsonld inbox signature check failed ' +
  2793. keyId)
  2794. if os.path.isfile(queueFilename):
  2795. os.remove(queueFilename)
  2796. if len(queue) > 0:
  2797. queue.pop(0)
  2798. continue
  2799. else:
  2800. if httpSignatureFailed:
  2801. print('jsonld inbox signature check success ' +
  2802. 'via relay ' + keyId)
  2803. else:
  2804. print('jsonld inbox signature check success ' + keyId)
  2805. # set the id to the same as the post filename
  2806. # This makes the filename and the id consistent
  2807. # if queueJson['post'].get('id'):
  2808. # queueJson['post']['id']=queueJson['id']
  2809. if _receiveUndo(session,
  2810. baseDir, httpPrefix, port,
  2811. sendThreads, postLog,
  2812. cachedWebfingers,
  2813. personCache,
  2814. queueJson['post'],
  2815. federationList,
  2816. debug):
  2817. print('Queue: Undo accepted from ' + keyId)
  2818. if os.path.isfile(queueFilename):
  2819. os.remove(queueFilename)
  2820. if len(queue) > 0:
  2821. queue.pop(0)
  2822. continue
  2823. if debug:
  2824. print('DEBUG: checking for follow requests')
  2825. if receiveFollowRequest(session,
  2826. baseDir, httpPrefix, port,
  2827. sendThreads, postLog,
  2828. cachedWebfingers,
  2829. personCache,
  2830. queueJson['post'],
  2831. federationList,
  2832. debug, projectVersion,
  2833. maxFollowers):
  2834. if os.path.isfile(queueFilename):
  2835. os.remove(queueFilename)
  2836. if len(queue) > 0:
  2837. queue.pop(0)
  2838. print('Queue: Follow activity for ' + keyId +
  2839. ' removed from queue')
  2840. continue
  2841. else:
  2842. if debug:
  2843. print('DEBUG: No follow requests')
  2844. if receiveAcceptReject(session,
  2845. baseDir, httpPrefix, domain, port,
  2846. sendThreads, postLog,
  2847. cachedWebfingers, personCache,
  2848. queueJson['post'],
  2849. federationList, debug):
  2850. print('Queue: Accept/Reject received from ' + keyId)
  2851. if os.path.isfile(queueFilename):
  2852. os.remove(queueFilename)
  2853. if len(queue) > 0:
  2854. queue.pop(0)
  2855. continue
  2856. if _receiveEventPost(recentPostsCache, session,
  2857. baseDir, httpPrefix,
  2858. domain, port,
  2859. sendThreads, postLog,
  2860. cachedWebfingers,
  2861. personCache,
  2862. queueJson['post'],
  2863. federationList,
  2864. queueJson['postNickname'],
  2865. debug):
  2866. print('Queue: Event activity accepted from ' + keyId)
  2867. if os.path.isfile(queueFilename):
  2868. os.remove(queueFilename)
  2869. if len(queue) > 0:
  2870. queue.pop(0)
  2871. continue
  2872. if _receiveUpdate(recentPostsCache, session,
  2873. baseDir, httpPrefix,
  2874. domain, port,
  2875. sendThreads, postLog,
  2876. cachedWebfingers,
  2877. personCache,
  2878. queueJson['post'],
  2879. federationList,
  2880. queueJson['postNickname'],
  2881. debug):
  2882. if debug:
  2883. print('Queue: Update accepted from ' + keyId)
  2884. if os.path.isfile(queueFilename):
  2885. os.remove(queueFilename)
  2886. if len(queue) > 0:
  2887. queue.pop(0)
  2888. continue
  2889. # get recipients list
  2890. recipientsDict, recipientsDictFollowers = \
  2891. _inboxPostRecipients(baseDir, queueJson['post'],
  2892. httpPrefix, domain, port, debug)
  2893. if len(recipientsDict.items()) == 0 and \
  2894. len(recipientsDictFollowers.items()) == 0:
  2895. if debug:
  2896. print('Queue: no recipients were resolved ' +
  2897. 'for post arriving in inbox')
  2898. if os.path.isfile(queueFilename):
  2899. os.remove(queueFilename)
  2900. if len(queue) > 0:
  2901. queue.pop(0)
  2902. continue
  2903. # if there are only a small number of followers then
  2904. # process them as if they were specifically
  2905. # addresses to particular accounts
  2906. noOfFollowItems = len(recipientsDictFollowers.items())
  2907. if noOfFollowItems > 0:
  2908. # always deliver to individual inboxes
  2909. if noOfFollowItems < 999999:
  2910. if debug:
  2911. print('DEBUG: moving ' + str(noOfFollowItems) +
  2912. ' inbox posts addressed to followers')
  2913. for handle, postItem in recipientsDictFollowers.items():
  2914. recipientsDict[handle] = postItem
  2915. recipientsDictFollowers = {}
  2916. # recipientsList = [recipientsDict, recipientsDictFollowers]
  2917. if debug:
  2918. print('*************************************')
  2919. print('Resolved recipients list:')
  2920. pprint(recipientsDict)
  2921. print('Resolved followers list:')
  2922. pprint(recipientsDictFollowers)
  2923. print('*************************************')
  2924. # Copy any posts addressed to followers into the shared inbox
  2925. # this avoid copying file multiple times to potentially many
  2926. # individual inboxes
  2927. if len(recipientsDictFollowers) > 0:
  2928. sharedInboxPostFilename = \
  2929. queueJson['destination'].replace(inboxHandle, inboxHandle)
  2930. if not os.path.isfile(sharedInboxPostFilename):
  2931. saveJson(queueJson['post'], sharedInboxPostFilename)
  2932. # for posts addressed to specific accounts
  2933. for handle, capsId in recipientsDict.items():
  2934. destination = \
  2935. queueJson['destination'].replace(inboxHandle, handle)
  2936. _inboxAfterInitial(recentPostsCache,
  2937. maxRecentPosts,
  2938. session, keyId, handle,
  2939. queueJson['post'],
  2940. baseDir, httpPrefix,
  2941. sendThreads, postLog,
  2942. cachedWebfingers,
  2943. personCache, queue,
  2944. domain,
  2945. onionDomain, i2pDomain,
  2946. port, proxyType,
  2947. federationList,
  2948. debug,
  2949. queueFilename, destination,
  2950. maxReplies, allowDeletion,
  2951. maxMentions, maxEmoji,
  2952. translate, unitTest,
  2953. YTReplacementDomain,
  2954. showPublishedDateOnly,
  2955. allowLocalNetworkAccess,
  2956. peertubeInstances,
  2957. lastBounceMessage,
  2958. themeName)
  2959. if debug:
  2960. pprint(queueJson['post'])
  2961. print('Queue: Queue post accepted')
  2962. if os.path.isfile(queueFilename):
  2963. os.remove(queueFilename)
  2964. if len(queue) > 0:
  2965. queue.pop(0)