mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-11-01 07:02:53 +00:00
289 lines
No EOL
8.7 KiB
TypeScript
289 lines
No EOL
8.7 KiB
TypeScript
// Twitter API data structures and transformation utilities
|
|
|
|
export interface Tweet {
|
|
__typename?: string;
|
|
lang?: string;
|
|
favorite_count: number;
|
|
created_at: string;
|
|
display_text_range?: [number, number];
|
|
entities: {
|
|
hashtags: Array<{
|
|
indices: [number, number];
|
|
text: string;
|
|
}>;
|
|
urls?: Array<{
|
|
display_url: string;
|
|
expanded_url: string;
|
|
indices: [number, number];
|
|
url: string;
|
|
}>;
|
|
user_mentions: Array<{
|
|
id_str: string;
|
|
indices: [number, number];
|
|
name: string;
|
|
screen_name: string;
|
|
}>;
|
|
symbols: Array<any>;
|
|
};
|
|
id_str: string;
|
|
text: string;
|
|
user: {
|
|
id_str: string;
|
|
name: string;
|
|
profile_image_url_https: string;
|
|
screen_name: string;
|
|
verified: boolean;
|
|
is_blue_verified?: boolean;
|
|
};
|
|
conversation_count: number;
|
|
photos?: Array<{
|
|
url: string;
|
|
width: number;
|
|
height: number;
|
|
}>;
|
|
videos?: Array<{
|
|
url: string;
|
|
thumbnail_url: string;
|
|
duration: number;
|
|
}>;
|
|
retweet_count?: number;
|
|
quote_count?: number;
|
|
reply_count?: number;
|
|
}
|
|
|
|
export interface TwitterAPIResponse {
|
|
data: {
|
|
bookmark_timeline_v2: {
|
|
timeline: {
|
|
instructions: Array<{
|
|
type: string;
|
|
entries?: Array<{
|
|
entryId: string;
|
|
sortIndex: string;
|
|
content: any;
|
|
}>;
|
|
}>;
|
|
};
|
|
};
|
|
};
|
|
}
|
|
|
|
// Twitter API features configuration
|
|
export const TWITTER_API_FEATURES = {
|
|
graphql_timeline_v2_bookmark_timeline: true,
|
|
responsive_web_graphql_exclude_directive_enabled: true,
|
|
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
responsive_web_enhance_cards_enabled: false,
|
|
rweb_tipjar_consumption_enabled: true,
|
|
responsive_web_twitter_article_notes_tab_enabled: true,
|
|
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
standardized_nudges_misinfo: true,
|
|
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
longform_notetweets_rich_text_read_enabled: true,
|
|
longform_notetweets_inline_media_enabled: true,
|
|
responsive_web_media_download_video_enabled: false,
|
|
responsive_web_text_conversations_enabled: false,
|
|
// Missing features that the API is complaining about
|
|
creator_subscriptions_quote_tweet_preview_enabled: true,
|
|
view_counts_everywhere_api_enabled: true,
|
|
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
tweetypie_unmention_optimization_enabled: true,
|
|
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
|
tweet_awards_web_tipping_enabled: true,
|
|
communities_web_enable_tweet_community_results_fetch: true,
|
|
responsive_web_edit_tweet_api_enabled: true,
|
|
longform_notetweets_consumption_enabled: true,
|
|
articles_preview_enabled: true,
|
|
rweb_video_timestamps_enabled: true,
|
|
verified_phone_label_enabled: true
|
|
};
|
|
|
|
export const BOOKMARKS_URL = `https://x.com/i/api/graphql/xLjCVTqYWz8CGSprLU349w/Bookmarks?features=${encodeURIComponent(JSON.stringify(TWITTER_API_FEATURES))}`;
|
|
|
|
/**
|
|
* Transform raw Twitter API response data into standardized Tweet format
|
|
*/
|
|
export function transformTweetData(input: any): Tweet | null {
|
|
try {
|
|
const tweet = input.content?.itemContent?.tweet_results?.result;
|
|
|
|
if (!tweet || !tweet.legacy) {
|
|
return null;
|
|
}
|
|
|
|
// Handle media entities
|
|
const media = tweet.legacy.entities?.media || [];
|
|
const photos = media
|
|
.filter((m: any) => m.type === 'photo')
|
|
.map((m: any) => ({
|
|
url: m.media_url_https,
|
|
width: m.sizes?.large?.w || 0,
|
|
height: m.sizes?.large?.h || 0,
|
|
}));
|
|
|
|
const videos = media
|
|
.filter((m: any) => m.type === 'video')
|
|
.map((m: any) => ({
|
|
url: m.video_info?.variants?.[0]?.url || '',
|
|
thumbnail_url: m.media_url_https,
|
|
duration: m.video_info?.duration_millis || 0,
|
|
}));
|
|
|
|
const transformed: Tweet = {
|
|
__typename: tweet.__typename,
|
|
lang: tweet.legacy?.lang,
|
|
favorite_count: tweet.legacy.favorite_count || 0,
|
|
created_at: new Date(tweet.legacy.created_at).toISOString(),
|
|
display_text_range: tweet.legacy.display_text_range,
|
|
entities: {
|
|
hashtags: tweet.legacy.entities?.hashtags || [],
|
|
urls: tweet.legacy.entities?.urls || [],
|
|
user_mentions: tweet.legacy.entities?.user_mentions || [],
|
|
symbols: tweet.legacy.entities?.symbols || [],
|
|
},
|
|
id_str: tweet.legacy.id_str,
|
|
text: tweet.legacy.full_text,
|
|
user: {
|
|
id_str: tweet.core?.user_results?.result?.legacy?.id_str || '',
|
|
name: tweet.core?.user_results?.result?.legacy?.name || 'Unknown',
|
|
profile_image_url_https: tweet.core?.user_results?.result?.legacy?.profile_image_url_https || '',
|
|
screen_name: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
|
|
verified: tweet.core?.user_results?.result?.legacy?.verified || false,
|
|
is_blue_verified: tweet.core?.user_results?.result?.is_blue_verified || false,
|
|
},
|
|
conversation_count: tweet.legacy.reply_count || 0,
|
|
retweet_count: tweet.legacy.retweet_count || 0,
|
|
quote_count: tweet.legacy.quote_count || 0,
|
|
reply_count: tweet.legacy.reply_count || 0,
|
|
};
|
|
|
|
if (photos.length > 0) {
|
|
transformed.photos = photos;
|
|
}
|
|
|
|
if (videos.length > 0) {
|
|
transformed.videos = videos;
|
|
}
|
|
|
|
return transformed;
|
|
} catch (error) {
|
|
console.error('Error transforming tweet data:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract all tweets from Twitter API response
|
|
*/
|
|
export function getAllTweets(data: TwitterAPIResponse): Tweet[] {
|
|
const tweets: Tweet[] = [];
|
|
|
|
try {
|
|
const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions || [];
|
|
|
|
for (const instruction of instructions) {
|
|
if (instruction.type === 'TimelineAddEntries' && instruction.entries) {
|
|
for (const entry of instruction.entries) {
|
|
if (entry.entryId.startsWith('tweet-')) {
|
|
const tweet = transformTweetData(entry);
|
|
if (tweet) {
|
|
tweets.push(tweet);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Error extracting tweets:', error);
|
|
}
|
|
|
|
return tweets;
|
|
}
|
|
|
|
/**
|
|
* Extract pagination cursor from Twitter API response
|
|
*/
|
|
export function extractNextCursor(instructions: any[]): string | null {
|
|
try {
|
|
for (const instruction of instructions) {
|
|
if (instruction.type === 'TimelineAddEntries' && instruction.entries) {
|
|
for (const entry of instruction.entries) {
|
|
if (entry.entryId.startsWith('cursor-bottom-')) {
|
|
return entry.content?.value || null;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Error extracting cursor:', error);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Convert Tweet object to markdown format for storage
|
|
*/
|
|
export function tweetToMarkdown(tweet: Tweet): string {
|
|
const username = tweet.user?.screen_name || 'unknown';
|
|
const displayName = tweet.user?.name || 'Unknown User';
|
|
const date = new Date(tweet.created_at).toLocaleDateString();
|
|
const time = new Date(tweet.created_at).toLocaleTimeString();
|
|
|
|
let markdown = `# Tweet by @${username} (${displayName})\n\n`;
|
|
markdown += `**Date:** ${date} ${time}\n`;
|
|
markdown += `**Likes:** ${tweet.favorite_count} | **Retweets:** ${tweet.retweet_count || 0} | **Replies:** ${tweet.reply_count || 0}\n\n`;
|
|
|
|
// Add tweet text
|
|
markdown += `${tweet.text}\n\n`;
|
|
|
|
// Add media if present
|
|
if (tweet.photos && tweet.photos.length > 0) {
|
|
markdown += `**Images:**\n`;
|
|
tweet.photos.forEach((photo, index) => {
|
|
markdown += `\n`;
|
|
});
|
|
markdown += '\n';
|
|
}
|
|
|
|
if (tweet.videos && tweet.videos.length > 0) {
|
|
markdown += `**Videos:**\n`;
|
|
tweet.videos.forEach((video, index) => {
|
|
markdown += `[Video ${index + 1}](${video.url})\n`;
|
|
});
|
|
markdown += '\n';
|
|
}
|
|
|
|
// Add hashtags and mentions
|
|
if (tweet.entities.hashtags.length > 0) {
|
|
markdown += `**Hashtags:** ${tweet.entities.hashtags.map(h => `#${h.text}`).join(', ')}\n`;
|
|
}
|
|
|
|
if (tweet.entities.user_mentions.length > 0) {
|
|
markdown += `**Mentions:** ${tweet.entities.user_mentions.map(m => `@${m.screen_name}`).join(', ')}\n`;
|
|
}
|
|
|
|
// Add raw data for reference
|
|
markdown += `\n---\n<details>\n<summary>Raw Tweet Data</summary>\n\n\`\`\`json\n${JSON.stringify(tweet, null, 2)}\n\`\`\`\n</details>`;
|
|
|
|
return markdown;
|
|
}
|
|
|
|
/**
|
|
* Build Twitter API request variables for pagination
|
|
*/
|
|
export function buildRequestVariables(cursor?: string, count: number = 100) {
|
|
const variables = {
|
|
count,
|
|
includePromotedContent: false,
|
|
};
|
|
|
|
if (cursor) {
|
|
(variables as any).cursor = cursor;
|
|
}
|
|
|
|
return variables;
|
|
} |