2d Unity Mirror + Steamworks | once the game starts, the playernames get mixed up

[solemn carbon]

Hi!
This is going to be a long post so I would appreciate if you could help me out with this..
So far I was able to create a lobby, get the steam ID of players who join and see them listed.
The host can then start the game and the player prefabs are able to spawn.
Here is the problem:
Every Client except the Host get their Names mixed up..

Example:
Lobby:
Player 1 (host)
Player 2
Player 3
Player 4

and once I start the game, it gets randomly assigned (except the host)
Player 1 (stays always the same)
Player 2 is now called Player 4
Player 4 is now called Player 3
Player 3 is now called Player 2
It’s completely random, I tried it couple a times, sometimes everyone gets their name correct.

The good thing is, I think I know what part of my code handles that, but I can't get it to work...
I have been trying to fix this issue for like 4 days straight..

  public override void OnServerAddPlayer(NetworkConnectionToClient conn)
  {
    base.OnServerAddPlayer(conn);

    MyNetworkPlayer player = conn.identity.GetComponent<MyNetworkPlayer>();
    Players.Add(player);
    
    CSteamID steamId = SteamMatchmaking.GetLobbyMemberByIndex(
      Menu.LobbyId, 
      numPlayers - 1);
    
      var playerInfoDisplay = conn.identity.GetComponent<MyNetworkPlayer>();
      playerInfoDisplay.SetSteamId(steamId.m_SteamID);

      // for (int i = 0; i < Players.Count; i++)
      // {
      //   player.SetPlayerName("Player " + i);
      // }

    player.SetPartyOwner(Players.Count == 1);
    
  }

If you need any more code snippets, please let me know
Thank you in advance

I'll just add all the code I have, in case you wanna look over it

part 1 of Mynetworkmanager

using System;
using System.Collections.Generic;
using UnityEngine;
using Mirror;
using Steamworks;

public class MyNetworkManager : NetworkManager
{
  

  public static event Action ClientOnConnected;
  public static event Action ClientOnDisconnected;

  private bool isGameInProgress;
  public List<MyNetworkPlayer> Players { get; } = new();

  

  #region Server

  public override void OnServerConnect(NetworkConnectionToClient conn)
  {
    if (!isGameInProgress) return;
    
    conn.Disconnect();
    
  }

  public override void OnServerDisconnect(NetworkConnectionToClient conn)
  {
    MyNetworkPlayer player = conn.identity.GetComponent<MyNetworkPlayer>();

    Players.Remove(player);
    
    base.OnServerDisconnect(conn);
  }

  public override void OnStopServer()
  {
    Players.Clear();

    isGameInProgress = false;
  }

  public void StartGame()
  {
    if (Players.Count < 1) return;

    isGameInProgress = true;
    
    ServerChangeScene("gameScene");
  }

  // ReSharper disable Unity.PerformanceAnalysis
  public override void OnServerAddPlayer(NetworkConnectionToClient conn)
  {
    base.OnServerAddPlayer(conn);

    MyNetworkPlayer player = conn.identity.GetComponent<MyNetworkPlayer>();
    Players.Add(player);
    
    CSteamID steamId = SteamMatchmaking.GetLobbyMemberByIndex(
      Menu.LobbyId, 
      numPlayers - 1);
    
      var playerInfoDisplay = conn.identity.GetComponent<MyNetworkPlayer>();
      playerInfoDisplay.SetSteamId(steamId.m_SteamID);

//      for (int i = 0; i < Players.Count; i++)
//      {
//        player.SetPlayerName("Player " + i);
//      }

    player.SetPartyOwner(Players.Count == 1);
    
  }

part 2 of mynetworkmanager

  public override void OnServerSceneChanged(string sceneName)
  {
    foreach (MyNetworkPlayer player in Players)
    {
      GameObject baseInstance = Instantiate(playerPrefab, GetStartPosition().position, Quaternion.identity);
      NetworkServer.Spawn(baseInstance, player.connectionToClient);
    }
  }

  #endregion
  
  #region Client
  public override void OnClientConnect()
  {
    base.OnClientConnect();
    ClientOnConnected?.Invoke();
  }

  public override void OnClientDisconnect()
  {
    base.OnClientDisconnect();
    ClientOnDisconnected?.Invoke();
  }

  public override void OnStopClient()
  {
    Players.Clear();
  }

  #endregion
}

this is the networkplayer:

part 1

using System;
using UnityEngine;
using Mirror;
using Steamworks;
using TMPro;


public class MyNetworkPlayer : NetworkBehaviour
{
    
    [SyncVar(hook = nameof(AuthorityHandlePartyOwnerStateUpdated))]
    private bool isPartyOwner = false;
    
    
    [SyncVar(hook = nameof(HandleSteamIdUpdated))] 
    private ulong steamID;

    [SerializeField] private TMP_Text playerNameText;
    

    public static event Action<bool> AuthorityOnPartyOwnerStateUpdated;
    public static event Action ClientOnInfoUpdated;
    
    

    public string GetDisplayName()
    {
        return playerNameText.text;
        
    }


    public void SetPlayerName(string nickName)
    {
        playerNameText.text = nickName;
    }
    
    
    public bool GetIsPartyOwner()
    {
        return isPartyOwner;
    }

    
    
    #region Server

    public void SetSteamId(ulong steamIdNum)
    {
        steamID = steamIdNum;
    }
    
    [Server]
    public void SetPartyOwner(bool state)
    {
        isPartyOwner = state;
    }

    #endregion

    #region Client

    public override void OnStartClient()
    {
        if (NetworkServer.active) return;
        ClientOnInfoUpdated?.Invoke();
        ((MyNetworkManager)NetworkManager.singleton).Players.Add(this);
    }

    public override void OnStopClient()
    {
        ClientOnInfoUpdated?.Invoke();
        
        if (isClientOnly) return;
        ((MyNetworkManager)NetworkManager.singleton).Players.Remove(this);

        if (!hasAuthority) return;
    }
    
    
    
    private void HandleSteamIdUpdated(ulong oldSteamId, ulong newSteamId)
    {
        ClientOnInfoUpdated?.Invoke();
        var cSteamId = new CSteamID(newSteamId);
        playerNameText.text = SteamFriends.GetFriendPersonaName(cSteamId);
        ClientOnInfoUpdated?.Invoke();

    }