From aaf7ceca860d856886bf7cf6e22bfa07a5a53d8c Mon Sep 17 00:00:00 2001 From: Vladyslav Khomenko Date: Fri, 31 Jan 2025 21:56:01 +0200 Subject: [PATCH 1/4] Added ability to enable the arcore geospatial mode and track the camera geospatial pose --- android/build.gradle | 12 +++++- .../lars/ar_flutter_plugin/AndroidARView.kt | 26 ++++++++++++- example/android/build.gradle | 4 +- example/ios/Podfile | 2 +- example/ios/Runner.xcodeproj/project.pbxproj | 7 +++- .../xcshareddata/xcschemes/Runner.xcscheme | 2 +- example/ios/Runner/AppDelegate.swift | 2 +- example/ios/Runner/Info.plist | 2 + ios/Classes/IosARView.swift | 38 +++++++++++++++++-- ios/ar_flutter_plugin.podspec | 3 +- lib/managers/ar_session_manager.dart | 28 ++++++++++++++ lib/models/ar_geospatial_pose.dart | 19 ++++++++++ pubspec.yaml | 4 +- 13 files changed, 133 insertions(+), 16 deletions(-) create mode 100644 lib/models/ar_geospatial_pose.dart diff --git a/android/build.gradle b/android/build.gradle index dbdff724..c5ee6056 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -2,7 +2,7 @@ group 'io.carius.lars.ar_flutter_plugin' version '1.0-SNAPSHOT' buildscript { - ext.kotlin_version = '1.3.50' + ext.kotlin_version = '1.5.20' repositories { google() jcenter() @@ -30,6 +30,14 @@ android { sourceSets { main.java.srcDirs += 'src/main/kotlin' } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8 + } defaultConfig { minSdkVersion 24 } @@ -37,7 +45,7 @@ android { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" - implementation "com.google.ar:core:1.22.0" + implementation "com.google.ar:core:1.47.0" implementation 'com.google.ar.sceneform:core:1.15.0' implementation 'com.google.ar.sceneform:assets:1.15.0' diff --git a/android/src/main/kotlin/io/carius/lars/ar_flutter_plugin/AndroidARView.kt b/android/src/main/kotlin/io/carius/lars/ar_flutter_plugin/AndroidARView.kt index 6ffb177b..9bf5025b 100644 --- a/android/src/main/kotlin/io/carius/lars/ar_flutter_plugin/AndroidARView.kt +++ b/android/src/main/kotlin/io/carius/lars/ar_flutter_plugin/AndroidARView.kt @@ -146,6 +146,14 @@ internal class AndroidARView( handlerThread.quitSafely(); }, Handler(handlerThread.looper)); } + "enableGeospatialMode" -> { + if(arSceneView.session?.isGeospatialModeSupported(Config.GeospatialMode.ENABLED) == true) { + val config = arSceneView.session?.config + config?.setGeospatialMode(Config.GeospatialMode.ENABLED) + arSceneView.session?.configure(config) + } + + } "dispose" -> { dispose() } @@ -657,7 +665,23 @@ internal class AndroidARView( //unselect all nodes as we do not want the selection visualizer transformationSystem.selectNode(null) } - + if(arSceneView.session?.config?.geospatialMode == Config.GeospatialMode.ENABLED) { + val session = arSceneView.session; + val earth = session?.earth + if (earth?.trackingState == TrackingState.TRACKING) { + val cameraGeospatialPose = earth.cameraGeospatialPose + val map: HashMap = HashMap() + map["latitude"] = cameraGeospatialPose.latitude + map["longitude"] = cameraGeospatialPose.longitude + map["altitude"] = cameraGeospatialPose.altitude + map["eastUpSouthQuaternion"] = cameraGeospatialPose.eastUpSouthQuaternion + map["horizontalAccuracy"] = cameraGeospatialPose.horizontalAccuracy + map["orientationYawAccuracy"] = cameraGeospatialPose.orientationYawAccuracy + map["verticalAccuracy"] = cameraGeospatialPose.verticalAccuracy + + sessionManagerChannel.invokeMethod("onCameraGeospatialPoseDetected", map) + } + } } private fun addNode(dict_node: HashMap, dict_anchor: HashMap? = null): CompletableFuture{ diff --git a/example/android/build.gradle b/example/android/build.gradle index d86df6bf..ae75c348 100644 --- a/example/android/build.gradle +++ b/example/android/build.gradle @@ -1,5 +1,5 @@ buildscript { - ext.kotlin_version = '1.6.10' + ext.kotlin_version = '1.7.0' repositories { google() jcenter() @@ -27,6 +27,6 @@ subprojects { project.evaluationDependsOn(':app') } -task clean(type: Delete) { +tasks.register("clean", Delete) { delete rootProject.buildDir } diff --git a/example/ios/Podfile b/example/ios/Podfile index 4b7408d8..7b86d6bc 100644 --- a/example/ios/Podfile +++ b/example/ios/Podfile @@ -1,5 +1,5 @@ # Uncomment this line to define a global platform for your project -# platform :ios, '9.0' +# platform :ios, '12.0' # Override firebase SDK version for compatibility reasons $FirebaseSDKVersion = '8.7.0' diff --git a/example/ios/Runner.xcodeproj/project.pbxproj b/example/ios/Runner.xcodeproj/project.pbxproj index 11a7fafa..2cf465bf 100644 --- a/example/ios/Runner.xcodeproj/project.pbxproj +++ b/example/ios/Runner.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 51; + objectVersion = 54; objects = { /* Begin PBXBuildFile section */ @@ -166,7 +166,7 @@ 97C146E61CF9000F007C117D /* Project object */ = { isa = PBXProject; attributes = { - LastUpgradeCheck = 1300; + LastUpgradeCheck = 1510; ORGANIZATIONNAME = ""; TargetAttributes = { 97C146ED1CF9000F007C117D = { @@ -212,10 +212,12 @@ /* Begin PBXShellScriptBuildPhase section */ 3B06AD1E1E4923F5004D2608 /* Thin Binary */ = { isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); inputPaths = ( + "${TARGET_BUILD_DIR}/${INFOPLIST_PATH}", ); name = "Thin Binary"; outputPaths = ( @@ -243,6 +245,7 @@ }; 9740EEB61CF901F6004384FC /* Run Script */ = { isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); diff --git a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index c87d15a3..5e31d3d3 100644 --- a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -1,6 +1,6 @@ This app needs access to location when in the background. CADisableMinimumFrameDurationOnPhone + UIApplicationSupportsIndirectInputEvents + diff --git a/ios/Classes/IosARView.swift b/ios/Classes/IosARView.swift index 49b239fc..b64cf271 100644 --- a/ios/Classes/IosARView.swift +++ b/ios/Classes/IosARView.swift @@ -4,6 +4,7 @@ import Foundation import ARKit import Combine import ARCoreCloudAnchors +import ARCoreGeospatial class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureRecognizerDelegate, ARSessionDelegate { let sceneView: ARSCNView @@ -74,7 +75,8 @@ class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureReco func onSessionMethodCalled(_ call :FlutterMethodCall, _ result:FlutterResult) { let arguments = call.arguments as? Dictionary - + + switch call.method { case "init": //self.sessionManagerChannel.invokeMethod("onError", arguments: ["SessionTEST from iOS"]) @@ -104,6 +106,22 @@ class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureReco } else { result(nil) } + case "enableGeospatialMode": + arcoreSession = try! GARSession(apiKey: arguments?["apiKey"] as! String, bundleIdentifier: nil) + if(!(arcoreSession?.isGeospatialModeSupported(GARGeospatialMode.enabled) ?? false)) { + result(false) + } + + if (arcoreSession != nil) { + let configuration = GARSessionConfiguration(); + configuration.geospatialMode = .enabled; + arcoreSession?.setConfiguration(configuration, error: nil); + arcoreMode = true + } else { + sessionManagerChannel.invokeMethod("onError", arguments: ["Error initializing Google AR Session"]) + } + + break case "dispose": onDispose(result) result(nil) @@ -182,7 +200,7 @@ class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureReco } break case "initGoogleCloudAnchorMode": - arcoreSession = try! GARSession.session() + arcoreSession = try! GARSession.session() if (arcoreSession != nil){ let configuration = GARSessionConfiguration(); @@ -366,7 +384,21 @@ class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureReco func session(_ session: ARSession, didUpdate frame: ARFrame) { if (arcoreMode) { do { - try arcoreSession!.update(frame) + let garFrame = try arcoreSession!.update(frame) + let earth = garFrame.earth + if(earth?.earthState == .enabled) { + let geospatialTransform = earth?.cameraGeospatialTransform + var map = Dictionary(); + map["latitude"] = geospatialTransform?.coordinate.latitude + map["longitude"] = geospatialTransform?.coordinate.longitude + map["altitude"] = geospatialTransform?.altitude + map["eastUpSouthQuaternion"] = geospatialTransform?.eastUpSouthQTarget + map["horizontalAccuracy"] = geospatialTransform?.horizontalAccuracy + map["orientationYawAccuracy"] = geospatialTransform?.orientationYawAccuracy + map["verticalAccuracy"] = geospatialTransform?.verticalAccuracy + + sessionManagerChannel.invokeMethod("onCameraGeospatialPoseDetected", arguments: map) + } } catch { print(error) } diff --git a/ios/ar_flutter_plugin.podspec b/ios/ar_flutter_plugin.podspec index 7ded92d5..12279959 100644 --- a/ios/ar_flutter_plugin.podspec +++ b/ios/ar_flutter_plugin.podspec @@ -20,7 +20,8 @@ A Flutter plugin for shared AR experiences supporting Android and iOS. s.static_framework = true #s.dependency 'ARCore/CloudAnchors', '~> 1.12.0' #s.dependency 'ARCore', '~> 1.2.0' - s.dependency 'ARCore/CloudAnchors', '~> 1.33.0' # Updated from 1.32 to 1.33 to support Apple Silicon, info here: https://github.com/google-ar/arcore-ios-sdk/issues/59#issuecomment-1219756010 + s.dependency 'ARCore/CloudAnchors', '~> 1.47.0' + s.dependency 'ARCore/Geospatial', '~> 1.47.0' s.platform = :ios, '13.0' diff --git a/lib/managers/ar_session_manager.dart b/lib/managers/ar_session_manager.dart index 20d3b83a..72d9ed16 100644 --- a/lib/managers/ar_session_manager.dart +++ b/lib/managers/ar_session_manager.dart @@ -3,6 +3,7 @@ import 'dart:typed_data'; import 'package:ar_flutter_plugin/datatypes/config_planedetection.dart'; import 'package:ar_flutter_plugin/models/ar_anchor.dart'; +import 'package:ar_flutter_plugin/models/ar_geospatial_pose.dart'; import 'package:ar_flutter_plugin/models/ar_hittest_result.dart'; import 'package:ar_flutter_plugin/utils/json_converters.dart'; import 'package:flutter/material.dart'; @@ -12,6 +13,9 @@ import 'package:vector_math/vector_math_64.dart'; // Type definitions to enforce a consistent use of the API typedef ARHitResultHandler = void Function(List hits); +typedef ARCameraGeospatialPoseHandler = void Function( + ARGeospatialPose cameraGeospatialPose); + /// Manages the session configuration, parameters and events of an [ARView] class ARSessionManager { /// Platform channel used for communication from and to [ARSessionManager] @@ -29,6 +33,8 @@ class ARSessionManager { /// Receives hit results from user taps with tracked planes or feature points late ARHitResultHandler onPlaneOrPointTap; + ARCameraGeospatialPoseHandler? onCameraGeospatialPoseDetected; + ARSessionManager(int id, this.buildContext, this.planeDetectionConfig, {this.debug = false}) { _channel = MethodChannel('arsession_$id'); @@ -103,6 +109,22 @@ class ARSessionManager { return distance; } + /// Enables geospatial mode for the application. +/// +/// This method invokes a platform channel to enable geospatial mode. +/// The `apiKey` parameter is **only required on iOS** and can be omitted for Android. +/// +/// - [apiKey]: (Optional) The API key required for iOS devices. +/// +/// Example usage: +/// ```dart +/// enableGeospatialMode(apiKey: 'your-ios-api-key'); // Required for iOS +/// enableGeospatialMode(); // Works on Android without API key +/// ``` + void enableGeospatialMode({String? apiKey}) { + _channel.invokeMethod('enableGeospatialMode', {'apiKey': apiKey}); + } + Future _platformCallHandler(MethodCall call) { if (debug) { print('_platformCallHandler call ${call.method} ${call.arguments}'); @@ -128,6 +150,12 @@ class ARSessionManager { onPlaneOrPointTap(hitTestResults); } break; + case 'onCameraGeospatialPoseDetected': + if (onCameraGeospatialPoseDetected != null) { + final pose = ARGeospatialPose.fromMap(call.arguments); + onCameraGeospatialPoseDetected!(pose); + } + break; case 'dispose': _channel.invokeMethod("dispose"); break; diff --git a/lib/models/ar_geospatial_pose.dart b/lib/models/ar_geospatial_pose.dart new file mode 100644 index 00000000..3bf0c4bd --- /dev/null +++ b/lib/models/ar_geospatial_pose.dart @@ -0,0 +1,19 @@ +class ARGeospatialPose { + double? latitude; + double? longitude; + double? altitude; + List? eastUpSouthQuaternion; + double? horizontalAccuracy; + double? orientationYawAccuracy; + double? verticalAccuracy; + + ARGeospatialPose.fromMap(Map map) { + this.latitude = map['latitude']; + this.longitude = map['longitude']; + this.altitude = map['altitude']; + this.eastUpSouthQuaternion = map['eastUpSouthQuaternion']; + this.horizontalAccuracy = map['horizontalAccuracy']; + this.orientationYawAccuracy = map['orientationYawAccuracy']; + this.verticalAccuracy = map['verticalAccuracy']; + } +} \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 8a25261f..b9c96d57 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -11,10 +11,10 @@ environment: dependencies: flutter: sdk: flutter - permission_handler: ^10.1.0 + permission_handler: ^11.3.1 vector_math: ^2.1.1 json_annotation: ^4.5.0 - geolocator: ^9.0.0 + geolocator: ^13.0.2 dev_dependencies: flutter_test: From be602fb46fda463857e3c821746150755720ef56 Mon Sep 17 00:00:00 2001 From: Vladyslav Khomenko Date: Fri, 31 Jan 2025 23:42:06 +0200 Subject: [PATCH 2/4] Replaced the ARCore pod with the ARCoreNanoPbUpdated --- ios/ar_flutter_plugin.podspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ios/ar_flutter_plugin.podspec b/ios/ar_flutter_plugin.podspec index 12279959..f8812c9b 100644 --- a/ios/ar_flutter_plugin.podspec +++ b/ios/ar_flutter_plugin.podspec @@ -20,8 +20,8 @@ A Flutter plugin for shared AR experiences supporting Android and iOS. s.static_framework = true #s.dependency 'ARCore/CloudAnchors', '~> 1.12.0' #s.dependency 'ARCore', '~> 1.2.0' - s.dependency 'ARCore/CloudAnchors', '~> 1.47.0' - s.dependency 'ARCore/Geospatial', '~> 1.47.0' + s.dependency 'ARCoreNanoPbUpdated/CloudAnchors', '~> 1.46.0.2' + s.dependency 'ARCoreNanoPbUpdated/Geospatial', '~> 1.46.0.2' s.platform = :ios, '13.0' From edff3b4d3303a06681d95072cdd643bb26fa0e9f Mon Sep 17 00:00:00 2001 From: Vladyslav Khomenko Date: Mon, 3 Feb 2025 16:43:38 +0200 Subject: [PATCH 3/4] Update AR session manager to support geospatial mode and improve quaternion handling --- android/build.gradle | 3 ++- ios/Classes/IosARView.swift | 3 ++- lib/managers/ar_session_manager.dart | 14 ++++++++------ lib/models/ar_geospatial_pose.dart | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index c5ee6056..8c0446ce 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -25,7 +25,8 @@ apply plugin: 'com.android.library' apply plugin: 'kotlin-android' android { - compileSdkVersion 30 + namespace 'io.carius.lars.ar_flutter_plugin' + compileSdkVersion 34 sourceSets { main.java.srcDirs += 'src/main/kotlin' diff --git a/ios/Classes/IosARView.swift b/ios/Classes/IosARView.swift index b64cf271..3ba00c9a 100644 --- a/ios/Classes/IosARView.swift +++ b/ios/Classes/IosARView.swift @@ -392,7 +392,8 @@ class IosARView: NSObject, FlutterPlatformView, ARSCNViewDelegate, UIGestureReco map["latitude"] = geospatialTransform?.coordinate.latitude map["longitude"] = geospatialTransform?.coordinate.longitude map["altitude"] = geospatialTransform?.altitude - map["eastUpSouthQuaternion"] = geospatialTransform?.eastUpSouthQTarget + var eastUpSouthQuaternion: [Float?] = [geospatialTransform?.eastUpSouthQTarget.imag.x, geospatialTransform?.eastUpSouthQTarget.imag.y, geospatialTransform?.eastUpSouthQTarget.imag.z, geospatialTransform?.eastUpSouthQTarget.real] + map["eastUpSouthQuaternion"] = eastUpSouthQuaternion map["horizontalAccuracy"] = geospatialTransform?.horizontalAccuracy map["orientationYawAccuracy"] = geospatialTransform?.orientationYawAccuracy map["verticalAccuracy"] = geospatialTransform?.verticalAccuracy diff --git a/lib/managers/ar_session_manager.dart b/lib/managers/ar_session_manager.dart index 72d9ed16..d9fa57f0 100644 --- a/lib/managers/ar_session_manager.dart +++ b/lib/managers/ar_session_manager.dart @@ -112,17 +112,16 @@ class ARSessionManager { /// Enables geospatial mode for the application. /// /// This method invokes a platform channel to enable geospatial mode. -/// The `apiKey` parameter is **only required on iOS** and can be omitted for Android. /// -/// - [apiKey]: (Optional) The API key required for iOS devices. +/// - [iosApiKey]: (Optional) The API key required for iOS devices. /// /// Example usage: /// ```dart -/// enableGeospatialMode(apiKey: 'your-ios-api-key'); // Required for iOS +/// enableGeospatialMode(iosApiKey: 'your-ios-api-key'); // Required for iOS /// enableGeospatialMode(); // Works on Android without API key /// ``` - void enableGeospatialMode({String? apiKey}) { - _channel.invokeMethod('enableGeospatialMode', {'apiKey': apiKey}); + void enableGeospatialMode({String? iosApiKey}) { + _channel.invokeMethod('enableGeospatialMode', {'apiKey': iosApiKey}); } Future _platformCallHandler(MethodCall call) { @@ -152,7 +151,10 @@ class ARSessionManager { break; case 'onCameraGeospatialPoseDetected': if (onCameraGeospatialPoseDetected != null) { - final pose = ARGeospatialPose.fromMap(call.arguments); + final rawPoseDetected = call.arguments as Map; + rawPoseDetected['eastUpSouthQuaternion'] = List.from( + rawPoseDetected['eastUpSouthQuaternion']); + final pose = ARGeospatialPose.fromMap(rawPoseDetected); onCameraGeospatialPoseDetected!(pose); } break; diff --git a/lib/models/ar_geospatial_pose.dart b/lib/models/ar_geospatial_pose.dart index 3bf0c4bd..f9a4a3a7 100644 --- a/lib/models/ar_geospatial_pose.dart +++ b/lib/models/ar_geospatial_pose.dart @@ -2,7 +2,7 @@ class ARGeospatialPose { double? latitude; double? longitude; double? altitude; - List? eastUpSouthQuaternion; + List? eastUpSouthQuaternion; double? horizontalAccuracy; double? orientationYawAccuracy; double? verticalAccuracy; From acf0c4a7759bc9573a07b95911758ef85253914b Mon Sep 17 00:00:00 2001 From: Vladyslav Khomenko Date: Mon, 3 Feb 2025 16:52:03 +0200 Subject: [PATCH 4/4] Bump version to 0.7.4 in pubspec.yaml --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index b9c96d57..e2b9e148 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: ar_flutter_plugin description: Flutter Plugin for creating (collaborative) Augmented Reality experiences - Supports ARKit for iOS and ARCore for Android devices. -version: 0.7.3 +version: 0.7.4 homepage: https://lars.carius.io repository: https://github.com/CariusLars/ar_flutter_plugin