Add text recognition

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

package-lock.json

@@ -12,6 +12,7 @@
         "@bankify/redux-persist-realm": "^0.1.3",
         "@react-native-clipboard/clipboard": "^1.11.2",
         "@react-native-community/hooks": "^3.0.0",
+        "@react-native-ml-kit/text-recognition": "^1.2.1",
         "@react-navigation/bottom-tabs": "^6.5.8",
         "@react-navigation/native": "^6.1.7",
         "@react-navigation/native-stack": "^6.9.13",
@@ -4113,6 +4114,15 @@
         "react-native": ">=0.65"
       }
     },
+    "node_modules/@react-native-ml-kit/text-recognition": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@react-native-ml-kit/text-recognition/-/text-recognition-1.2.1.tgz",
+      "integrity": "sha512-pJrnf8AvihzYdPAZoZZEeKbOUOMjdsetDjHlleXOoVcoPo6qjfh6Il/Q0ey3boIQuO3HglvNjcMPGEPThF3sPA==",
+      "peerDependencies": {
+        "react": ">=16.8.1",
+        "react-native": ">=0.60.0-rc.0 <1.0.x"
+      }
+    },
     "node_modules/@react-native/assets-registry": {
       "version": "0.72.0",
       "resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.72.0.tgz",
@@ -18799,6 +18809,12 @@
       "integrity": "sha512-g2OyxXHfwIytXUJitBR6Z/ISoOfp0WKx5FOv+NqJ/CrWjRDcTw6zXE5I1C9axfuh30kJqzWchVfCDrkzZYTxqg==",
       "requires": {}
     },
+    "@react-native-ml-kit/text-recognition": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@react-native-ml-kit/text-recognition/-/text-recognition-1.2.1.tgz",
+      "integrity": "sha512-pJrnf8AvihzYdPAZoZZEeKbOUOMjdsetDjHlleXOoVcoPo6qjfh6Il/Q0ey3boIQuO3HglvNjcMPGEPThF3sPA==",
+      "requires": {}
+    },
     "@react-native/assets-registry": {
       "version": "0.72.0",
       "resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.72.0.tgz",

package.json

@@ -17,6 +17,7 @@
     "@bankify/redux-persist-realm": "^0.1.3",
     "@react-native-clipboard/clipboard": "^1.11.2",
     "@react-native-community/hooks": "^3.0.0",
+    "@react-native-ml-kit/text-recognition": "^1.2.1",
     "@react-navigation/bottom-tabs": "^6.5.8",
     "@react-navigation/native": "^6.1.7",
     "@react-navigation/native-stack": "^6.9.13",

src/components/index.ts

@@ -5,3 +5,4 @@ export { default as HideableHeader } from './hideableHeader';
 export { default as LoadingView } from './loadingView';
 export { default as MemeFail } from './memeFail';
 export { default as TagChip } from './tagChip';
+export { default as TextOverlay } from './textOverlay';

src/components/textOverlay.tsx

@@ -0,0 +1,64 @@
+import React from 'react';
+import { TextBlock } from '@react-native-ml-kit/text-recognition';
+import { TouchableRipple, useTheme } from 'react-native-paper';
+import { StyleSheet } from 'react-native';
+import { Dimensions } from '../types';
+
+const textOverlayStyles = StyleSheet.create({
+  touchable: {
+    position: 'absolute',
+    borderWidth: 1,
+    zIndex: 1,
+  },
+});
+
+const TextOverlay = ({
+  blocks,
+  onTextPress,
+  onTextLongPress,
+  imageDimensions,
+  frameDimensions,
+}: {
+  blocks: TextBlock[];
+  onTextPress: (text: string) => void;
+  onTextLongPress: (text: string) => void;
+  imageDimensions: Dimensions;
+  frameDimensions: Dimensions;
+}) => {
+  const { colors } = useTheme();
+
+  const widthScale = frameDimensions.width / imageDimensions.width;
+  const heightScale = frameDimensions.height / imageDimensions.height;
+
+  return (
+    <>
+      {blocks.map(
+        (block, index) =>
+          block.frame && (
+            <TouchableRipple
+              key={index}
+              style={[
+                textOverlayStyles.touchable,
+                {
+                  top: block.frame.top * heightScale - 5,
+                  left: block.frame.left * widthScale - 5,
+                  width: block.frame.width * widthScale + 10,
+                  height: block.frame.height * heightScale + 10,
+                  borderColor: colors.error,
+                },
+              ]}
+              onPress={() =>
+                onTextPress(block.text.replaceAll('\n', ' ').trim())
+              }
+              onLongPress={() =>
+                onTextLongPress(block.text.replaceAll('\n', ' ').trim())
+              }>
+              <></>
+            </TouchableRipple>
+          ),
+      )}
+    </>
+  );
+};
+
+export default TextOverlay;

src/screens/editors/meme/memeEditor.tsx

@@ -1,9 +1,9 @@
-import React, { useMemo } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
 import { HelperText, Text, TextInput, useTheme } from 'react-native-paper';
-import { Image, LayoutAnimation } from 'react-native';
+import { Image, LayoutAnimation, View } from 'react-native';
 import { useSafeAreaFrame } from 'react-native-safe-area-context';
 import Video from 'react-native-video';
-import { LoadingView, MemeFail } from '../../../components';
+import { LoadingView, MemeFail, TextOverlay } from '../../../components';
 import {
   getFilenameFromUri,
   getMemeTypeFromMimeType,
@@ -13,6 +13,9 @@ import { StagingMeme } from '../../../types';
 import { useMemeDimensions } from '../../../hooks';
 import { MEME_TYPE } from '../../../database';
 import MemeTagSelector from './memeTagSelector/memeTagSelector';
+import TextRecognition, {
+  TextRecognitionResult,
+} from '@react-native-ml-kit/text-recognition';
 
 const memeEditorStyles = {
   media: {
@@ -66,8 +69,15 @@ const MemeEditor = ({
     useMemo(() => (errorIn: Error) => setError(errorIn), [setError]),
   );
 
+  const [recognizedText, setRecognizedText] = useState<TextRecognitionResult>();
+
+  useEffect(() => {
+    if (!uri || !mimeType || !mimeType.startsWith('image')) return;
+    void TextRecognition.recognize(uri).then(setRecognizedText);
+  }, [mimeType, uri]);
+
   const mediaComponent = useMemo(() => {
-    if (!mimeType || !dimensions) return <></>;
+    if (!mimeType || !dimensions || !staging) return <></>;
 
     const dimensionStyles = {
       width: width * 0.92,
@@ -84,11 +94,35 @@ const MemeEditor = ({
       case MEME_TYPE.IMAGE:
       case MEME_TYPE.GIF: {
         return (
-          <Image
-            source={{ uri }}
-            style={[memeEditorStyles.media, dimensionStyles]}
-            resizeMode="contain"
-          />
+          <View>
+            <Image
+              source={{ uri }}
+              style={[memeEditorStyles.media, dimensionStyles]}
+              resizeMode="contain"
+            />
+            {recognizedText && (
+              <TextOverlay
+                blocks={recognizedText.blocks}
+                onTextPress={text =>
+                  setStaging({
+                    ...staging,
+                    title: validateMemeTitle(text),
+                  })
+                }
+                onTextLongPress={text =>
+                  setStaging({
+                    ...staging,
+                    title: validateMemeTitle(`${staging.title.parsed} ${text}`),
+                  })
+                }
+                imageDimensions={dimensions}
+                frameDimensions={{
+                  ...dimensionStyles,
+                  aspectRatio: dimensionStyles.width / dimensionStyles.height,
+                }}
+              />
+            )}
+          </View>
         );
       }
       case MEME_TYPE.VIDEO: {
@@ -105,7 +139,7 @@ const MemeEditor = ({
         return <></>;
       }
     }
-  }, [dimensions, mimeType, uri, width]);
+  }, [dimensions, mimeType, recognizedText, setStaging, staging, uri, width]);
 
   if (!uri || !mimeType || !staging) return <LoadingView />;